Merge branch 'stanfordnlp:main' into main

Author: itay1542

.github/.tmp/.generated-actions/run-pypi-publish-in-docker-container/action.yml

@@ -0,0 +1 @@
+{"name": "🏃", "description": "Run Docker container to upload Python distribution packages to PyPI", "inputs": {"user": {"description": "PyPI user", "required": false}, "password": {"description": "Password for your PyPI user or an access token", "required": false}, "repository-url": {"description": "The repository URL to use", "required": false}, "packages-dir": {"description": "The target directory for distribution", "required": false}, "verify-metadata": {"description": "Check metadata before uploading", "required": false}, "skip-existing": {"description": "Do not fail if a Python package distribution exists in the target package index", "required": false}, "verbose": {"description": "Show verbose output.", "required": false}, "print-hash": {"description": "Show hash values of files to be uploaded", "required": false}, "attestations": {"description": "[EXPERIMENTAL] Enable experimental support for PEP 740 attestations. Only works with PyPI and TestPyPI via Trusted Publishing.", "required": false}}, "runs": {"using": "docker", "image": "docker://ghcr.io/pypa/gh-action-pypi-publish:release-v1"}}

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,38 @@
+
+name: Bug Report
+description: Report a bug in the project
+title: "[Bug] "
+labels: bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## 🐛 Bug Report
+        Please fill out all required fields to help us diagnose and fix the issue.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: "What happened?"
+      description: "Clearly describe the unexpected behavior."
+      placeholder: "Example: When I try to save a file, I get an error message..."
+    validations:
+      required: true
+
+  - type: textarea
+    id: steps-to-reproduce
+    attributes:
+      label: "Steps to reproduce"
+      description: "Tell us how to reproduce the issue."
+      placeholder: "Please provide a code snippet or a github gist for reproducing purpose."
+    validations:
+      required: true
+
+  - type: input
+    id: environment
+    attributes:
+      label: "DSPy version"
+      description: "Tell us your DSPy version."
+    validations:
+      required: true
+

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,33 @@
+name: Feature Request
+description: Suggest a new feature or improvement
+title: "[Feature] "
+labels: enhancement
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## 🚀 Feature Request
+        Please fill out the following details.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: "What feature would you like to see?"
+      description: "Describe the feature clearly."
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: contribute
+    attributes:
+      label: "Would you like to contribute?"
+      options:
+        - label: Yes, I'd like to help implement this.
+        - label: No, I just want to request it.
+
+  - type: textarea
+    id: additional-info
+    attributes:
+      label: "Additional Context"
+      description: "Any links, references, or extra details?"
+      placeholder: "Example: This feature exists in XYZ tool."

.github/workflows/build_and_release.yml

@@ -23,14 +23,15 @@ jobs:
       name: pypi
     permissions:
       id-token: write  # IMPORTANT: mandatory for trusted publishing
+      contents: write
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.9
         uses: actions/setup-python@v3
         with:
           python-version: "3.9"
       - name: Install dependencies
-        run: python3 -m pip install setuptools wheel twine semver packaging
+        run: python3 -m pip install --upgrade setuptools wheel twine semver packaging
       - name: Get correct version for TestPyPI release  
         id: check_version  
         run: |  
@@ -41,11 +42,11 @@ jobs:
           echo "Version to be used for TestPyPI release: $NEW_VERSION"  
           echo "::set-output name=version::$NEW_VERSION"  
       - name: Update version in setup.py
-        run: sed -i '/#replace_package_version_marker/{n;s/version="[^"]*"/version="${{ steps.check_version.outputs.version }}"/;}' setup.py
+        run: sed -i '/#replace_package_version_marker/{n;s/__version__="[^"]*"/__version__="${{ steps.check_version.outputs.version }}"/;}' ./dspy/__metadata__.py
       - name: Update version in pyproject.toml
         run: sed -i '/#replace_package_version_marker/{n;s/version="[^"]*"/version="${{ steps.check_version.outputs.version }}"/;}' pyproject.toml        
       - name: Update package name in setup.py
-        run: sed -i '/#replace_package_name_marker/{n;s/name="[^"]*"/name="dspy-ai-test"/;}' setup.py 
+        run: sed -i '/#replace_package_name_marker/{n;s/__name__="[^"]*"/__name__="dspy-ai-test"/;}' ./dspy/__metadata__.py
       - name: Update package name in pyproject.toml
         run: sed -i '/#replace_package_name_marker/{n;s/name="[^"]*"/name="dspy-ai-test"/;}' pyproject.toml  
       - name: Build a binary wheel
@@ -62,41 +63,67 @@ jobs:
       name: pypi
     permissions:
       id-token: write  # IMPORTANT: mandatory for trusted publishing
+      contents: write
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.9
         uses: actions/setup-python@v3
         with:
           python-version: "3.9"
       - name: Install dependencies
-        run: python3 -m pip install setuptools wheel twine
+        run: python3 -m pip install --upgrade setuptools wheel twine
       - name: Update version in setup.py (dspy)
-        run: sed -i '/#replace_package_version_marker/{n;s/version="[^"]*"/version="${{ needs.extract-tag.outputs.version }}"/;}' setup.py
+        run: sed -i '/#replace_package_version_marker/{n;s/__version__ *= *"[^"]*"/__version__="${{ needs.extract-tag.outputs.version }}"/;}' ./dspy/__metadata__.py
       - name: Update version in pyproject.toml
-        run: sed -i '/#replace_package_version_marker/{n;s/version="[^"]*"/version="${{ needs.extract-tag.outputs.version }}"/;}' pyproject.toml
+        run: sed -i '/#replace_package_version_marker/{n;s/version *= *"[^"]*"/version="${{ needs.extract-tag.outputs.version }}"/;}' pyproject.toml
       # Publish to dspy
       - name: Update package name in setup.py  
         run: |
-          sed -i '/#replace_package_name_marker/{n;s/name="[^"]*"/name="dspy"/;}' setup.py
+          sed -i '/#replace_package_name_marker/{n;s/__name__ *= *"[^"]*"/__name__="dspy"/;}' ./dspy/__metadata__.py
       - name: Update package name in pyproject.toml
-        run: sed -i '/#replace_package_name_marker/{n;s/name="[^"]*"/name="dspy"/;}' pyproject.toml
+        run: sed -i '/#replace_package_name_marker/{n;s/name *= *"[^"]*"/name="dspy"/;}' pyproject.toml
+      # Remove pyproject.toml temporarily to avoid conflicts
+      - name: Temporarily remove pyproject.toml  
+        run: mv pyproject.toml pyproject.toml.bak 
       - name: Build a binary wheel
         run: python3 setup.py sdist bdist_wheel
       - name: Publish distribution 📦 to PyPI (dspy)
         uses: pypa/gh-action-pypi-publish@release/v1 # This requires a trusted publisher to be setup in pypi
         with: 
           attestations: false
-        # Publish to dspy-ai
+      # Publish to dspy-ai
       - name: Update version in setup.py (dspy-ai)
-        run: sed -i '/#replace_package_version_marker/{n;s/version="[^"]*"/version="${{ needs.extract-tag.outputs.version }}"/;}' ./dspy/.internal_dspyai/setup.py
+        run: sed -i '/#replace_package_version_marker/{n;s/version *= *"[^"]*"/version="${{ needs.extract-tag.outputs.version }}"/;}' ./dspy/.internal_dspyai/setup.py
       - name: Update package name in setup.py  
-        run: sed -i '/#replace_package_name_marker/{n;s/name="[^"]*"/name="dspy-ai"/;}' ./dspy/.internal_dspyai/setup.py
+        run: sed -i '/#replace_package_name_marker/{n;s/name *= *"[^"]*"/name="dspy-ai"/;}' ./dspy/.internal_dspyai/setup.py
       - name: Update dspy dependency version in setup.py  
         run: |
-          sed -i '/#replace_dspy_version_marker/{n;s/dspy==[^"]*/dspy>=${{ needs.extract-tag.outputs.version }}/;}' ./dspy/.internal_dspyai/setup.py  
+          sed -i '/#replace_dspy_version_marker/{n;s/dspy>=[^"]*/dspy>=${{ needs.extract-tag.outputs.version }}/;}' ./dspy/.internal_dspyai/setup.py  
       - name: Build a binary wheel
         run: python3 ./dspy/.internal_dspyai/setup.py sdist bdist_wheel
       - name: Publish distribution 📦 to PyPI (dspy-ai)
         uses: pypa/gh-action-pypi-publish@release/v1 # This requires a trusted publisher to be setup in pypi
         with: 
-          attestations: false
+          attestations: false
+      # Restore pyproject.toml
+      - name: Restore pyproject.toml  
+        run: mv pyproject.toml.bak pyproject.toml 
+      - uses: stefanzweifel/git-auto-commit-action@v5 # auto commit changes to main
+        with:
+          commit_message: Update versions
+          create_branch: true
+          branch: release-${{ needs.extract-tag.outputs.version }}
+      - name: Checkout main branch  
+        run: |  
+          git fetch origin  
+          git checkout main  
+      - name: Configure git user
+        run: | 
+          git config --global user.email "[email protected]"
+          git config --global user.name "Github Actions"
+      - name: Merge release branch into main  
+        run: |  
+          git merge --no-ff release-${{ needs.extract-tag.outputs.version }}          
+      - name: Push changes to main
+        run: |  
+          git push origin main  

.github/workflows/docs-push.yml

@@ -15,7 +15,8 @@ jobs:
     if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v3
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:

.github/workflows/run_tests.yml

@@ -8,7 +8,7 @@ on:
     types: [opened, synchronize, reopened]
 
 env:
-  POETRY_VERSION: "1.7.1"
+  POETRY_VERSION: "2.0.0"
 
 jobs:
   fix:
@@ -41,28 +41,36 @@ jobs:
         python-version: ["3.9"]
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Deno
+        run: |
+          curl -fsSL https://deno.land/install.sh | sh
+          echo "Deno installed"
+
+      - name: Add Deno to PATH
+        run: echo "${HOME}/.deno/bin" >> $GITHUB_PATH
+
+      - name: Verify Deno installation
+        run: deno --version
+
       - name: Load cached Poetry installation
         id: cached-poetry
         uses: actions/cache@v3
         with:
           path: ~/.local
           key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
       - name: Install Poetry
-        if: steps.cached-poetry.outputs.cache-hit != 'true'
-        uses: snok/install-poetry@v1
-      - name: Set up python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
+        run: python -m pip install --upgrade "poetry==${{ env.POETRY_VERSION }}"
       - name: Install dependencies
         run: poetry install --no-interaction
       - name: Run lint with tests
         uses: chartboost/ruff-action@v1
         with:
           args: check --fix-only
       - name: Run tests with pytest
-        run: poetry run pytest tests/ --ignore=tests/reliability
+        run: poetry run pytest tests/
 
   build_poetry:
     name: Build Poetry
@@ -78,14 +86,11 @@ jobs:
         with:
           path: ~/.local
           key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
-      - name: Install Poetry
-        if: steps.cached-poetry.outputs.cache-hit != 'true'
-        uses: snok/install-poetry@v1
-      - name: Set up python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "poetry"
+      - name: Install Poetry
+        run: python -m pip install --upgrade "poetry==${{ env.POETRY_VERSION }}"
       - name: Build
         run: poetry build
       - name: Install built package
@@ -107,13 +112,10 @@ jobs:
         with:
           path: ~/.local
           key: poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
-      - name: Install Poetry
-        if: steps.cached-poetry.outputs.cache-hit != 'true'
-        uses: snok/install-poetry@v1
-      - name: Set up python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "poetry"
+      - name: Install Poetry
+        run: python -m pip install --upgrade "poetry==${{ env.POETRY_VERSION }}"
       - name: Run setup.py build
         run: python setup.py build

.gitignore

@@ -62,3 +62,4 @@ dummy.csv
 docs/docs/**/*.json*
 *.index
 *.pkl
+*.tar.gz

.prettierignore

@@ -0,0 +1 @@
+.github/

README.md

@@ -8,7 +8,7 @@
 
 **Documentation:** [DSPy Docs](https://dspy.ai/)
 
-[![Downloads](https://static.pepy.tech/badge/dspy-ai)](https://pepy.tech/project/dspy-ai)  [![Downloads](https://static.pepy.tech/badge/dspy/month)](https://pepy.tech/project/dspy)
+[![PyPI Downloads](https://static.pepy.tech/badge/dspy)](https://pepy.tech/projects/dspy)  [![PyPI Downloads](https://static.pepy.tech/badge/dspy/month)](https://pepy.tech/projects/dspy)
 
 
 ----
@@ -42,6 +42,10 @@ pip install git+https://github.com/stanfordnlp/dspy.git
 
 ## 📜 Citation & Reading More
 
+If you're looking to understand the framework, please go to the [DSPy Docs at dspy.ai](https://dspy.ai).
+
+If you're looking to understand the underlying research, this is a set of our papers:
+
 **[Jun'24] [Optimizing Instructions and Demonstrations for Multi-Stage Language Model Programs](https://arxiv.org/abs/2406.11695)**       
 **[Oct'23] [DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines](https://arxiv.org/abs/2310.03714)**     
 [Jul'24] [Fine-Tuning and Prompt Optimization: Two Great Steps that Work Better Together](https://arxiv.org/abs/2407.10930)     

docs/README.md

@@ -16,7 +16,14 @@ To build and test the documentation locally:
    pip install -r requirements.txt
    ```
 
-3. Run the build command:
+3. In docs/ directory, run the command below to generate the API docs and index them:
+   ```bash
+   cd docs/
+   python scripts/generate_api_docs.py
+   python scripts/generate_api_summary.py
+   ```
+
+4. Run the build command:
    ```bash
    mkdocs build
    ```

docs/docs/api/adapters/Adapter.md

@@ -0,0 +1,19 @@
+# dspy.Adapter
+
+::: dspy.Adapter
+    handler: python
+    options:
+        members:
+            - __call__
+            - format
+            - format_finetune_data
+            - parse
+        show_source: true
+        show_undocumented_members: true
+        show_root_heading: true
+        show_inherited_members: true
+        heading_level: 2
+        docstring_style: google
+        show_root_full_path: true
+        show_object_full_path: false
+        separate_signature: false

docs/docs/api/adapters/ChatAdapter.md

@@ -0,0 +1,21 @@
+# dspy.ChatAdapter
+
+::: dspy.ChatAdapter
+    handler: python
+    options:
+        members:
+            - __call__
+            - format
+            - format_fields
+            - format_finetune_data
+            - format_turn
+            - parse
+        show_source: true
+        show_undocumented_members: true
+        show_root_heading: true
+        show_inherited_members: true
+        heading_level: 2
+        docstring_style: google
+        show_root_full_path: true
+        show_object_full_path: false
+        separate_signature: false

docs/docs/api/adapters/JSONAdapter.md

@@ -0,0 +1,21 @@
+# dspy.JSONAdapter
+
+::: dspy.JSONAdapter
+    handler: python
+    options:
+        members:
+            - __call__
+            - format
+            - format_fields
+            - format_finetune_data
+            - format_turn
+            - parse
+        show_source: true
+        show_undocumented_members: true
+        show_root_heading: true
+        show_inherited_members: true
+        heading_level: 2
+        docstring_style: google
+        show_root_full_path: true
+        show_object_full_path: false
+        separate_signature: false