31 ci benchmarking

.github/workflows/benchmark_pr.yml

@@ -1,78 +1,122 @@
-name: Benchmark a pull request
+name: GridTools Benchmark CI Pipeline
 
 on:
-  pull_request:
+  push:
     branches:
       - main
+    tags: ['*']
+  pull_request:
 
-permissions:
-  pull-requests: write
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 
 jobs:
-    generate_plots:
-        runs-on: ubuntu-latest
+  benchmark:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        julia_version:
+          - '1.8'
+        python_version:
+          - '3.10'
+        os:
+          - ubuntu-latest
+        arch:
+          - x64
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python_version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python_version }}
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libboost-all-dev
+          python -m pip install --upgrade pip
+
+      - name: Install GT4Py and atlas4py
+        run: |
+          git clone --branch fix_python_interp_path_in_cmake https://github.com/tehrengruber/gt4py.git
+          cd gt4py
+          pip install -r requirements-dev.txt
+          pip install .
+          pip install -i https://test.pypi.org/simple/ atlas4py
+
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.julia_version }}
+          arch: ${{ matrix.arch }}
+
+      - uses: julia-actions/cache@v1
+
+      - name: Install and Build Benchmarking Tools
+        run: |
+          julia -e 'using Pkg; Pkg.add("AirspeedVelocity"); Pkg.build("AirspeedVelocity")'
+          echo "PATH=$PATH:$HOME/.julia/bin" >> $GITHUB_ENV
+          ls $HOME/.julia/bin
 
-        steps:
-            - uses: actions/checkout@v2
-            - uses: julia-actions/setup-julia@v1
-              with:
-                version: "1.8"
-            - uses: julia-actions/cache@v1
-            - name: Extract Package Name from Project.toml
-              id: extract-package-name
-              run: |
-                PACKAGE_NAME=$(grep "^name" Project.toml | sed 's/^name = "\(.*\)"$/\1/')
-                echo "::set-output name=package_name::$PACKAGE_NAME"
-            - name: Build AirspeedVelocity
-              env:
-                JULIA_NUM_THREADS: 2
-              run: |
-                # Lightweight build step, as sometimes the runner runs out of memory:
-                julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.add(;url="https://github.com/MilesCranmer/AirspeedVelocity.jl.git")'
-                julia -e 'ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0; import Pkg; Pkg.build("AirspeedVelocity")'
-            - name: Add ~/.julia/bin to PATH
-              run: |
-                echo "$HOME/.julia/bin" >> $GITHUB_PATH
-            - name: Run benchmarks
-              run: |
-                echo $PATH
-                ls -l ~/.julia/bin
-                mkdir results
-                benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.repository.default_branch}}" --output-dir=results/ --tune
-            - name: Create plots from benchmarks
-              run: |
-                mkdir -p plots
-                benchpkgplot ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
-            - name: Upload plot as artifact
-              uses: actions/upload-artifact@v2
-              with:
-                name: plots
-                path: plots
-            - name: Create markdown table from benchmarks
-              run: |
-                benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md
-                echo '### Benchmark Results' > body.md
-                echo '' >> body.md
-                echo '' >> body.md
-                cat table.md >> body.md
-                echo '' >> body.md
-                echo '' >> body.md
-                echo '### Benchmark Plots' >> body.md
-                echo 'A plot of the benchmark results have been uploaded as an artifact to the workflow run for this PR.' >> body.md
-                echo 'Go to "Actions"->"Benchmark a pull request"->[the most recent run]->"Artifacts" (at the bottom).' >> body.md
+      - name: Run Benchmarks
+        run: |
+          echo "Current PATH: $PATH"
+          mkdir results
+          git fetch origin main:refs/remotes/origin/main
+          CURRENT_COMMIT=$(git rev-parse HEAD)
+          LAST_MAIN_COMMIT=$(git rev-parse origin/main)
+          echo "Benchmarking current commit ($CURRENT_COMMIT) in the current branch and ($LAST_MAIN_COMMIT) in the main branch"
+          benchpkg --rev="$LAST_MAIN_COMMIT,$CURRENT_COMMIT" --bench-on="$CURRENT_COMMIT" --output-dir=results/ --tune
+
+      - name: Generate and Upload Benchmark Plots
+        run: |
+          mkdir -p plots
+          CURRENT_COMMIT=$(git rev-parse HEAD)
+          LAST_MAIN_COMMIT=$(git rev-parse origin/main)
+          echo "Generating plots comparing current commit ($CURRENT_COMMIT) against ($LAST_MAIN_COMMIT) in the main branch"
+          benchpkgplot GridTools --rev="$LAST_MAIN_COMMIT,$CURRENT_COMMIT" --npart=10 --format=png --input-dir=results/ --output-dir=plots/
+
+      - name: Upload Plots as Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-plots
+          path: plots
+
+      - name: Create and Display Benchmark Table
+        run: |
+          mkdir -p results
+          CURRENT_COMMIT=$(git rev-parse HEAD)
+          LAST_MAIN_COMMIT=$(git rev-parse origin/main)  # Fetch the latest commit from main for comparison
+          echo "Creating benchmark table comparing current commit ($CURRENT_COMMIT) against ($LAST_MAIN_COMMIT) in the main branch"
+          benchpkgtable --rev="$LAST_MAIN_COMMIT,$CURRENT_COMMIT" --input-dir=results/ --ratio > table.md
+          echo '### Benchmark Results' > body.md
+          cat table.md >> body.md
+          echo '### Benchmark Plots' >> body.md
+          echo 'A plot of the benchmark results has been uploaded as an artifact to this workflow run.' >> body.md
+          cat body.md  # Print the markdown table to the log for review.
+
+      - name: Upload Benchmark Results Table
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results-table
+          path: body.md
 
-            # - name: Find Comment
-            #   uses: peter-evans/find-comment@v2
-            #   id: fcbenchmark
-            #   with:
-            #     issue-number: ${{ github.event.pull_request.number }}
-            #     comment-author: 'github-actions[bot]'
-            #     body-includes: Benchmark Results
+      - name: Find and Comment Benchmark Results
+        uses: peter-evans/find-comment@v2
+        id: fcbenchmark
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Benchmark Results
 
-            - name: Comment on PR
-              uses: peter-evans/create-or-update-comment@v3
-              with:
-                # comment-id: ${{ steps.fcbenchmark.outputs.comment-id }}
-                issue-number: ${{ github.event.pull_request.number }}
-                body-path: body.md
-                edit-mode: replace
+      - name: Comment on PR with Benchmark Results
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          comment-id: ${{ steps.fcbenchmark.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-path: body.md
+          edit-mode: replace

.github/workflows/documentation.yml

@@ -1,10 +1,10 @@
-name: Documentation
+name: Documentation Build
 
 on:
   push:
     branches:
-      - main # update to match your development branch (master, main, dev, trunk, ...)
-    tags: '*'
+      - main # Reflects your active development branch
+    tags: ['*']
   pull_request:
 
 jobs:
@@ -13,14 +13,31 @@ jobs:
       contents: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+
+      - name: Set up Python environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install python3-pip
+          python3 -m pip install --upgrade pip
+
+      - name: Install GT4Py and atlas4py
+        run: |
+          git clone --branch fix_python_interp_path_in_cmake https://github.com/tehrengruber/gt4py.git
+          cd gt4py
+          pip install -r requirements-dev.txt
+          pip install .
+          pip install -i https://test.pypi.org/simple/ atlas4py
+
       - uses: julia-actions/setup-julia@v1
         with:
           version: '1.8'
-      - name: Install dependencies
+
+      - name: Install Julia dependencies
         run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
-      - name: Build and deploy
+
+      - name: Build and deploy documentation
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # If authenticating with GitHub Actions token
-          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If authenticating with SSH deploy key
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Used for Git operations by Documenter.jl
+          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # If using SSH deploy keys for secure operations
         run: julia --project=docs/ docs/make.jl

.gitignore

@@ -18,3 +18,17 @@ docs/build/
 .DS_Store
 
 Manifest.toml
+
+# Python Env
+.venv
+env_setup.sh
+.python-version
+
+# Misc
+**/.DS_Store
+.vscode
+
+# Ignore benchmark (benchpkg) results
+results_GridTools@*
+plot_*.png
+plot_*.pdf

Project.toml

@@ -12,6 +12,7 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
+Polyester = "f517fe37-dbe3-4b94-8317-1923a5111588"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
 PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"

advection/README.md

@@ -1,6 +1,6 @@
-### README for Running `advection_miniapp.jl`
+### README for Running `advection_setup.jl` using `run_simulation_loop.jl`
 
-This README provides instructions on how to run the `advection_miniapp.jl` script for simulating advection using the Atlas library. The script allows for terminal visualization, which can be enabled as described below.
+This README provides instructions on how to run the `run_simulation_loop.jl` script for simulating advection using the Atlas library. The script allows for terminal visualization, which can be enabled as described below.
 
 #### Prerequisites
 
@@ -15,23 +15,23 @@ This README provides instructions on how to run the `advection_miniapp.jl` scrip
      ```
 
 2. **Enabling Visualization** (optional):
-   - The script has a `VISUALIZATION_FLAG` that can be set to enable or disable visualization on the terminal. Ensure that this flag is set to `true` in the `advection_miniapp.jl` script if you wish to enable visualization.
-   - Note: Other parameters such as the number of iterations can be changed in the `# Simulation Parameters` section of the script.
+   - The script has a `VISUALIZATION_FLAG` that can be set to enable or disable visualization on the terminal. Ensure that this flag is set to `true` in the `run_simulation_loop.jl` script if you wish to enable visualization.
+   - Note: Other parameters such as the number of iterations can be changed in the `# Simulation Parameters` section of the `advection_setup.jl` script.
 
 #### Running the Simulation
 
 1. **Running the Script**:
-   - Use the following command to run the `advection_miniapp.jl` script with Julia:
+   - Use the following command to run the `run_simulation_loop.jl` script with Julia:
      ```sh
-     julia --color=yes --project=$GRIDTOOLS_JL_PATH/GridTools.jl $GRIDTOOLS_JL_PATH/GridTools.jl/src/examples/advection/advection_miniapp.jl
+     julia --color=yes --project=$GRIDTOOLS_JL_PATH/GridTools.jl $GRIDTOOLS_JL_PATH/GridTools.jl/src/examples/advection/run_simulation_loop.jl
      ```
 
 #### Example
 
-Here is an example of how to set the `VISUALIZATION_FLAG` in the `advection_miniapp.jl` script and run the simulation:
+Here is an example of how to set the `VISUALIZATION_FLAG` in the `run_simulation_loop.jl` script and run the simulation:
 
 1. **Setting the Visualization Flag**:
-   - Open the `advection_miniapp.jl` script.
+   - Open the `run_simulation_loop.jl` script.
    - Set the `VISUALIZATION_FLAG` to `true`:
      ```julia
      const VISUALIZATION_FLAG = true
@@ -42,7 +42,7 @@ Here is an example of how to set the `VISUALIZATION_FLAG` in the `advection_mini
    - Run the script with the following command:
      ```sh
      export GRIDTOOLS_JL_PATH=...
-     julia --color=yes --project=. $GRIDTOOLS_JL_PATH/src/examples/advection/advection_miniapp.jl
+     julia --color=yes --project=. $GRIDTOOLS_JL_PATH/src/examples/advection/run_simulation_loop.jl
      ```
 
-By following these steps, you should be able to run the `advection_miniapp.jl` script and visualize the advection simulation results on your terminal.
+By following these steps, you should be able to run the `run_simulation_loop.jl` script and visualize the advection simulation results on your terminal.

advection/advection.jl

@@ -6,11 +6,10 @@
     level_indices::Field{Tuple{K_}, Int64},
     num_level::Int64
 )::Field{Tuple{Vertex_, K_}, Float64}
-
     return where(
-        level_indices .== num_level - 1,
+        level_indices .== 0,
         lower,
-        where(slice(level_indices .== 0, 1:29), upper, interior)
+        where(slice(level_indices .== 29, 2:30), upper, interior)
     )
 end
 
@@ -149,7 +148,8 @@ end
 )::Field{Tuple{Vertex_, K_}, Float64}
     zrhin =
         (1.0 ./ vol) .* neighbor_sum(
-            -min.(0.0, flux(V2E)) .* max.(0.0, dual_face_orientation) -
+            # TODO: fix the 0-min workaround due to the binary/unary operation issue
+            (broadcast(0., (Vertex, V2EDim, K)) .- min.(0.0, flux(V2E))) .* max.(0.0, dual_face_orientation) -
             max.(0.0, flux(V2E)) .* min.(0.0, dual_face_orientation),
             axis = V2EDim,
         )
@@ -227,15 +227,6 @@ end
     dual_face_orientation::Field{Tuple{Vertex_, V2EDim_}, Float64},
     dual_face_normal_weighted_x::Field{Tuple{Edge_}, Float64},
     dual_face_normal_weighted_y::Field{Tuple{Edge_}, Float64},
-    tmp_vertex_1::Field{Tuple{Vertex_, K_}, Float64},
-    tmp_vertex_2::Field{Tuple{Vertex_, K_}, Float64},
-    tmp_vertex_3::Field{Tuple{Vertex_, K_}, Float64},
-    tmp_vertex_4::Field{Tuple{Vertex_, K_}, Float64},
-    tmp_vertex_5::Field{Tuple{Vertex_, K_}, Float64},
-    tmp_vertex_6::Field{Tuple{Vertex_, K_}, Float64},
-    tmp_edge_1::Field{Tuple{Edge_, K_}, Float64},
-    tmp_edge_2::Field{Tuple{Edge_, K_}, Float64},
-    tmp_edge_3::Field{Tuple{Edge_, K_}, Float64},
 )
 
     tmp_edge_1 = advector_normal(