Merge pull request #18 from numpex/17-update-benchmark-methodology

update benchmark methodology

CITATION.cff

@@ -1,13 +1,73 @@
 cff-version: 1.2.0 
 message: If you use this software, please cite it as below.
 authors:
+- family-names: Alliez
+  given-names: Pierre
+  affiliation: "INRIA"
+  email: [email protected]
+- family-names: Chabannes
+  given-names: Vincent
+  affiliation: "University of Strasbourg"
+  email: [email protected]
+- family-names: Faucher
+  given-names: Florian
+  affiliation: "INRIA"
+  email: [email protected]
+- family-names: Franck
+  given-names: Emmanuel
+  affiliation: "INRIA"
+  email: [email protected]
+- family-names: Georgiadis
+  given-names: Christos
+  affiliation: "INRIA"
+  email: [email protected]
+- family-names: Gouarin
+  given-names: Loic
+  affiliation: "École Polytechnique"
+  email: [email protected]
+- family-names: Hecht
+  given-names: Frederic
+  affiliation: "Sorbonne Université"
+  email: [email protected]
+- family-names: Jamond
+  given-names: Olivier
+  affiliation: "CEA"
+  email: [email protected]
+- family-names: Jolivet
+  given-names: Pierre
+  affiliation: "Sorbonne Université"
+  email: [email protected]
+- family-names: Jolivet
+  given-names: Pierre
+  affiliation: "CNRS"
+  email: [email protected]
+- family-names: Marait
+  given-names: Gilles
+  affiliation: "INRIA"
+  email: [email protected]
+- family-names: Melab
+  given-names: Nouredine
+  affiliation: "University of Lille"
+  email: [email protected]
+- family-names: Perlat
+  given-names: Jean-Philippe
+  affiliation: "CEA"
+  email: [email protected]
 - family-names: Prud'homme
   given-names: Christophe
   orcid: 'https://orcid.org/0000-0003-2287-2961'
   affiliation: "University of Strasbourg"
   email: [email protected]
+- family-names: Talbi
+  given-names: El-Ghazali
+  affiliation: "University of Lille"
+  email: [email protected]
+- family-names: Tournier
+  given-names: Pierre-Henri
+  affiliation: "Sorbonne Université"
+  email: [email protected]
 title: "numpex/exa-ma-d7.1 - Benchmarking Analysis Report"
-version: 0.0.1
+version: 0.0.6
 abstract: >
   This repository contains the deliverable D7.1 for the Benchmarking Analysis Report 
   of the Exa-MA project. 
@@ -16,7 +76,7 @@ keywords:
   - Exascale
   - Benchmarking
 license: CC-BY-4.0
-date-released: 2024-08-19
+date-released: 2024-09-27
 url: "https://github.com/numpex/exa-ma-d7.1"
 references:
   - type: grant

a.cli

@@ -8,6 +8,7 @@ DEFAULT_ZOTERO_USER_ID=""
 DEFAULT_ZOTERO_API_KEY=""
 DEFAULT_ZOTERO_GROUP_ID="5582837"
 DEFAULT_BIBTEX_FILE="references.bib"
+COLLECTION_ID=""
 
 # Read environment variables or set defaults
 ZOTERO_USER_ID=${ZOTERO_USER_ID:-$DEFAULT_ZOTERO_USER_ID}
@@ -149,9 +150,28 @@ update_bibtex() {
     else
         URL="https://api.zotero.org/groups/$ZOTERO_GROUP_ID/items?format=bibtex"
     fi
+    start=0
+    limit=100
+    echo "" > "$BIBTEX_FILE"
+    while $has_more; do
+        response=$(curl -s -H "Zotero-API-Key: $ZOTERO_API_KEY" "$URL&start=$start&limit=$limit")
+
+        if [ -z "$response" ]; then
+            echo "No more items to fetch."
+            has_more=false
+            break
+        fi
+
+        echo "$response" >> "$BIBTEX_FILE"
 
-    # Fetch and save BibTeX entries
-    curl -s -H "Zotero-API-Key: $ZOTERO_API_KEY" "$URL" > "$BIBTEX_FILE"
+        # Check if we need to fetch more items
+        num_items=$(echo "$response" | grep -c "@")  # Counting the number of BibTeX entries
+        if [ "$num_items" -lt "$limit" ]; then
+            has_more=false
+        else
+            start=$((start + limit))
+        fi
+    done
 
     echo "BibTeX entries updated successfully in $BIBTEX_FILE."
 }

chapters/WP1/00-index.tex

@@ -2,5 +2,4 @@
 \input{software/feelpp/WP1/WP1.tex}
 \input{software/freefempp/WP1/WP1.tex}
 \input{software/hawen/WP1/WP1.tex}
-\input{software/mmg-parmmg/WP1/WP1.tex}
 \input{software/samurai/WP1/WP1.tex}

chapters/WP3/00-index.tex

@@ -3,6 +3,6 @@
 \input{software/feelpp/WP3/WP3.tex}
 \input{software/freefempp/WP3/WP3.tex}
 \input{software/hawen/WP3/WP3.tex}
-\input{software/hpdomain-decomposition-methods/WP3/WP3.tex}
+\input{software/hpddm/WP3/WP3.tex}
 \input{software/manta/WP3/WP3.tex}
 \input{software/trust-platform/WP3/WP3.tex}

chapters/software.tex

@@ -2,110 +2,189 @@
 \chapter{Software}
 \label{sec:software}
 
-    This chapter presents the software developed within Exa-MA, focusing on features, mathematics, functionalities, publications, acknowledgments, and contact details.
-
-    First, we present some general statistics about the software identified for Exa-MA that will present some benchmarking results. 
-    Other software (new or existing) will be included in the future version of this document.
-
-        \paragraph{Architectures}
-        The following pie chart~\ref{fig:arch} shows the distribution of hardware architectures used.
-        \begin{figure}[H]
+This chapter presents the software developed within Exa-MA, focusing on features, mathematics, functionalities, publications, acknowledgments, and contact details.
+
+First, we present some general statistics about the software identified for Exa-MA that will present some benchmarking results. 
+Other software (new or existing) will be included in the future versions of this document.
+
+In the context of the D7.1 deliverable, it is essential to clarify the supported features across different hardware configurations, specifically CPUs, GPUs, and hybrid setups. 
+The following classification describes the architectural capabilities that will be benchmarked:
+
+\paragraph{CPU Only}
+Software that will exclusively run on CPU architectures (e.g., CGAL, Freefem++, Feel++, Manta). Benchmarks for these codes will only target CPU architectures.
+
+\paragraph{GPU Only}
+Software that will exclusively run on GPU architectures (e.g., Zellij). Benchmarks for these codes will focus solely on GPU architectures.
+
+\paragraph{CPU or GPU}
+Software that can run on either CPU or GPU but not both simultaneously (e.g., PyTorch, SciMba). In this case, benchmarks will be performed on both CPU and GPU, but no single run will execute on both architectures simultaneously. Some benchmarks may be CPU-only, others GPU-only, depending on the software's capabilities.
+
+\paragraph{CPU and GPU}
+Software that supports simultaneous execution on both CPU and GPU during a single simulation run (e.g., TRUST). Benchmarks in this category will involve executing components on both CPU and GPU, ensuring that at least one computational component runs on each architecture.
+
+\paragraph{Explanation of Benchmarking Criteria}
+\begin{itemize}
+    \item \textbf{CPU Only:} If selected, benchmarks will be performed exclusively on CPU architectures.
+    \item \textbf{GPU Only:} If selected, benchmarks will focus exclusively on GPU architectures.
+    \item \textbf{CPU and GPU:} If selected, benchmarks will involve simultaneous execution on CPU and GPU within the same run.
+    \item \textbf{CPU or GPU:} If selected, benchmarks will involve execution on either CPU or GPU, but not simultaneously. Some benchmarks will target CPU, others GPU, depending on the computational components.
+\end{itemize}
+
+
+\section{General Statistics}
+\label{sec:software:statistics}
+
+In this section, we provide an overview of the key characteristics and technological choices for the software developed and benchmarked within Exa-MA. 
+These statistics offer insights into the diversity of hardware architectures, programming languages, and parallel computing technologies utilized across the different software packages. 
+
+The aim is to highlight the widespread usage of various technologies, demonstrating both the flexibility and the breadth of approaches within the project. 
+Additionally, the DevOps practices employed, such as continuous integration, testing, and deployment, are presented to underscore the commitment to ensuring quality, reliability, and maintainability of the software developed under Exa-MA.
+
+The following subsections shows different aspects of the software involved in the project, from supported architectures and programming languages to specific parallelism technologies, data formats, and DevOps strategies. 
+It helps to assess the readiness of the software for large-scale simulations and benchmarks in exascale computing environments.
+
+
+\subsection{Architectures}
+
+The following pie chart~\ref{fig:arch} shows the distribution of hardware architectures used for the benchmarks.
+
+\begin{figure}[H]
 \centering
 \begin{tikzpicture}
-\pie[text=legend, color={red, orange, yellow}, sum=auto]{13/CPU, 7/GPU, 1/Hybrid}
+\pie[text=legend, color={red, orange, yellow, lime}, sum=auto]{9/CPU Only, 5/CPU or GPU, 1/CPU and GPU, 2/GPU Only}
 \end{tikzpicture}
-\caption{Distribution of hardware architectures}
+\caption{Distribution of benchmarked hardware architectures}
 \label{fig:arch}
 \end{figure}
 
-        \paragraph{Programming Languages}
-        The following pie chart~\ref{fig:languages} shows the distribution of programming languages used, highlighting the variety of computational solutions employed.
-        \begin{figure}[H]
+
+\subsection{Programming Languages}
+
+The following pie chart~\ref{fig:languages} shows the distribution of programming languages used, highlighting the variety of computational solutions employed.
+
+\begin{figure}[H]
 \centering
 \begin{tikzpicture}
-\pie[text=legend, color={red, orange, yellow, lime, skyblue, pink, cyan, magenta}, sum=auto]{11/C++, 1/C\#, 3/C, 4/Fortran, 2/C++17, 4/Python, 1/C++20, 1/C++14}
+\pie[text=legend, color={red, orange, yellow, lime, skyblue, pink, cyan, magenta}, sum=auto]{11/C++, 1/C\#, 2/C, 3/Fortran, 2/C++17, 4/Python, 1/C++20, 1/C++14}
 \end{tikzpicture}
 \caption{Distribution of programming languages}
 \label{fig:languages}
 \end{figure}
 
-        \paragraph{Parallelism Technology}
-        The pie chart~\ref{fig:parallelism} below represents the parallelism techniques used in Exa-MA software selected for this document.
-        \begin{figure}[H]
+
+\subsection{Parallelism Technology}
+
+
+The pie chart~\ref{fig:parallelism} below represents the parallelism techniques used in Exa-MA software selected for this document.
+
+\begin{figure}[H]
 \centering
 \begin{tikzpicture}
-\pie[text=legend, color={red, orange, yellow, lime, skyblue, pink}, sum=auto]{7/Multithread, 13/MPI, 6/GPU, 1/Parallelism - C++, 1/Task based, 1/Chapel}
+\pie[text=legend, color={red, orange, yellow, lime, skyblue, pink}, sum=auto]{7/Multithread, 12/MPI, 7/GPU, 1/Parallelism - C++, 1/Task based, 1/Chapel}
 \end{tikzpicture}
 \caption{Distribution of parallelism technologies}
 \label{fig:parallelism}
 \end{figure}
 
-        \paragraph{Data Formats}
-        The chart~\ref{fig:data} shows the supported data formats, for flexibility and compatibility in data handling, supported by Exa-MA software selected for this document.
-        \begin{figure}[H]
+
+
+\subsection{Data Formats}
+
+The chart~\ref{fig:data} shows the supported data formats, for flexibility and compatibility in data handling, supported by Exa-MA software selected for this document.
+
+\begin{figure}[H]
 \centering
 \begin{tikzpicture}
-\pie[text=legend, color={red, orange, yellow, lime, skyblue, pink, cyan, magenta, peach, lavender}, sum=auto]{2/XML, 5/HDF5, 2/JSON, 2/Ensight, 5/None, 1/YAML, 1/Data-management system, 6/VTK, 6/in-house format, 4/Gmsh and associated formats, 2/MED, 1/MFront}
+\pie[text=legend, color={red, orange, yellow, lime, skyblue, pink, cyan, magenta, peach, lavender}, sum=auto]{2/XML, 5/HDF5, 2/JSON, 2/Ensight, 5/None, 1/YAML, 1/Data-management system, 6/VTK, 5/in-house format, 4/Gmsh and associated formats, 2/MED, 1/MFront}
 \end{tikzpicture}
 \caption{Distribution of data formats}
 \label{fig:data}
 \end{figure}
 
-        \paragraph{DevOps - CI/CD}
-        The pie chart~\ref{fig:devops-cicd} below displays the support of continuous integration and deployment practices as well as continuous benchmarking, showcasing systematic software updates, quality maintenance and performance regression.
-        \begin{figure}[H]
+
+\subsection{Resilience}
+
+The pie chart~\ref{fig:resilience} below shows the resilience mechanisms used in Exa-MA software selected for this document.
+
+\begin{figure}[H]
+\centering
+\begin{tikzpicture}
+\pie[text=legend, color={red}, sum=auto]{6/Checkpoint restart}
+\end{tikzpicture}
+\caption{Distribution of Resilience strategies}
+\label{fig:resilience}
+\end{figure}
+
+
+
+\subsection{DevOps - CI/CD}
+
+The pie chart~\ref{fig:devops-cicd} below displays the support of continuous integration and deployment practices as well as continuous benchmarking, showcasing systematic software updates, quality maintenance and performance regression.
+
+\begin{figure}[H]
 \centering
 \begin{tikzpicture}
-\pie[text=legend, color={red, orange, yellow, lime}, sum=auto]{11/Continuous Integration, 1/Continuous Benchmarking, 1/Continuous Delivery, 4/None}
+\pie[text=legend, color={red, orange, yellow, lime}, sum=auto]{10/Continuous Integration, 1/Continuous Benchmarking, 2/Continuous Delivery, 4/None}
 \end{tikzpicture}
 \caption{Distribution of DevOps CI/CD/CD}
 \label{fig:devops-cicd}
 \end{figure}
 
-        \paragraph{DevOps - Packaging}
-        The next chart~\ref{fig:devops-packaging} shows different packaging methods used, which help in the distribution and management of software.
-        \begin{figure}[H]
+
+\subsection{DevOps - Packaging}
+
+The next chart~\ref{fig:devops-packaging} shows different packaging methods used, which help in the distribution and management of software.
+
+\begin{figure}[H]
 \centering
 \begin{tikzpicture}
-\pie[text=legend, color={red, orange, yellow, lime, skyblue, pink}, sum=auto]{10/None, 5/Debian-based, 2/Fedora, 2/Spack, 1/GUIX, 1/Other}
+\pie[text=legend, color={red, orange, yellow, lime, skyblue, pink}, sum=auto]{9/None, 5/Debian-based, 2/Fedora, 2/Spack, 1/GUIX, 1/Other}
 \end{tikzpicture}
 \caption{Distribution of DevOps Packaging}
 \label{fig:devops-packaging}
 \end{figure}
 
-        \paragraph{DevOps - Containers}
-        The pie chart~\ref{fig:devops-containers} displays the use of container technologies, which help encapsulate the software to run reliably in various environments.
-        \begin{figure}[H]
+
+\subsection{DevOps - Containers}
+
+The pie chart~\ref{fig:devops-containers} displays the use of container technologies, which help encapsulate the software to run reliably in various environments.
+
+\begin{figure}[H]
 \centering
 \begin{tikzpicture}
-\pie[text=legend, color={red, orange, yellow}, sum=auto]{12/None, 2/Apptainer/Singularity, 2/Docker}
+\pie[text=legend, color={red, orange, yellow}, sum=auto]{11/None, 2/Apptainer/Singularity, 2/Docker}
 \end{tikzpicture}
 \caption{Distribution of DevOps Containers}
 \label{fig:devops-containers}
 \end{figure}
 
-        \paragraph{DevOps - Testing}
-        The following pie chart~\ref{fig:devops-testing} details the testing practices adopted, illustrating the commitment to software reliability and functionality.
-        \begin{figure}[H]
+
+\subsection{DevOps - Testing}
+
+The following pie chart~\ref{fig:devops-testing} details the testing practices adopted, illustrating the commitment to software reliability and functionality.
+
+\begin{figure}[H]
 \centering
 \begin{tikzpicture}
-\pie[text=legend, color={red, orange, yellow, lime}, sum=auto]{8/Unit, 7/Verification, 3/Validation, 6/None}
+\pie[text=legend, color={red, orange, yellow, lime, skyblue}, sum=auto]{8/Unit, 7/Verification, 3/Validation, 5/None, 1/Functional}
 \end{tikzpicture}
 \caption{Distribution of DevOps Testing}
 \label{fig:devops-testing}
 \end{figure}
+
+
 \input{software/arcane-framework/arcane-framework.tex}
 \input{software/cgal/cgal.tex}
 \input{software/composyx/composyx.tex}
 \input{software/feelpp/feelpp.tex}
 \input{software/freefempp/freefempp.tex}
 \input{software/hawen/hawen.tex}
-\input{software/hpdomain-decomposition-methods/hpdomain-decomposition-methods.tex}
+\input{software/hpddm/hpddm.tex}
 \input{software/mahyco/mahyco.tex}
 \input{software/manta/manta.tex}
-\input{software/mmg-parmmg/mmg-parmmg.tex}
 \input{software/pbb/pbb.tex}
 \input{software/samurai/samurai.tex}
 \input{software/scimba/scimba.tex}
 \input{software/trust-platform/trust-platform.tex}
 \input{software/zellij/zellij.tex}
+

defs.tex

@@ -1,4 +1,8 @@
 \def\todo#1{{\color{numpexred}\textbf{TODO:} ``#1''}}
 \def\hyperride{Exa-MA}
-\def\exama{Exa-MA}
-\def\numpex{NumPEx}
+\def\exama{\ac{exama}\xspace}
+\def\exasoft{\textsc{Exa-Soft}\xspace}
+\def\exadost{\textsc{Exa-Dost}\xspace}
+\def\exaatow{\textsc{Exa-Atow}\xspace}
+\def\exadi{\textsc{Exa-DI}\xspace}
+\def\numpex{NumPEx}