kernels(mpi): enable WAMR support (#30)

* examples(mpi-kernels): enable mpi kernels with latest faasm version

* gha: bump faasm version again

* chore: remove unnecessary run container

* deps(faasmctl): bump to version 0.28.0

* chore: add pyyaml to requirements.txt

Author: csegarragonz

.github/workflows/release.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Get the code"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
       - name: "Set version variables"
@@ -34,17 +34,17 @@ jobs:
           echo "PYTHON_VERSION=$PYTHON_VER" >> $GITHUB_ENV
           echo "EXAMPLES_VERSION=${CPP_VER}_${PYTHON_VER}" >> $GITHUB_ENV
       - name: "Set up QEMU"
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: "Set up Docker Buildx"
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: "Log in to DockerHub"
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: faasm.azurecr.io
           username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }}
           password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }}
       - name: "Build and push examples-build image"
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           push: true
           file: docker/build.dockerfile
@@ -54,13 +54,3 @@ jobs:
             PYTHON_VERSION=${{ env.PYTHON_VERSION }}
           context: .
           tags: faasm.azurecr.io/examples-build:${{ env.EXAMPLES_VERSION }}
-      - name: "Build and push examples-run image"
-        uses: docker/build-push-action@v3
-        with:
-          push: true
-          file: docker/run.dockerfile
-          build-args: |
-            EXAMPLES_VERSION=${{ env.EXAMPLES_VERSION }}
-            FAASM_VERSION=${{ env.FAASM_VERSION }}
-          context: .
-          tags: faasm.azurecr.io/examples-run:${{ env.FAASM_VERSION }}

.github/workflows/tests.yml

@@ -41,7 +41,7 @@ jobs:
       - name: "Checkout code"
         uses: actions/checkout@v4
       # Check if any of the submodules have been modified
-      - uses: dorny/paths-filter@v2
+      - uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
@@ -57,7 +57,7 @@ jobs:
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
           echo "cpp-commit=$(git submodule status ./cpp | cut '-d ' -f 2)" >> $GITHUB_OUTPUT
           echo "py-commit=$(git submodule status ./python | cut '-d ' -f 2)" >> $GITHUB_OUTPUT
-      - uses: actions/cache/restore@v3.3.1
+      - uses: actions/cache/restore@v4
         id: wasm-cache
         with:
           path: ./wasm
@@ -109,7 +109,7 @@ jobs:
           mv /usr/local/faasm/wasm ./wasm
       # If we are here we _always_ want to overwrite the cache
       - name: "Prepare WASM cache"
-        uses: actions/cache/save@v3.3.1
+        uses: actions/cache/save@v4
         if: always()
         with:
           path: ./wasm
@@ -133,7 +133,7 @@ jobs:
     env:
       CONAN_CACHE_MOUNT_SOURCE: ~/.conan
       FAASM_INI_FILE: ./faasm.ini
-      FAASM_VERSION: 0.20.1
+      FAASM_VERSION: 0.21.1
       FAASM_WASM_VM: ${{ matrix.faasm_wasm_vm }}
     steps:
       - uses: csegarragonz/set-compose-version-action@main
@@ -175,7 +175,7 @@ jobs:
           echo "cpp-commit=$(git submodule status ./cpp | cut '-d ' -f 2)" >> $GITHUB_OUTPUT
           echo "py-commit=$(git submodule status ./python | cut '-d ' -f 2)" >> $GITHUB_OUTPUT
       - name: "Get WASM cache"
-        uses: actions/cache/restore@v3.3.1
+        uses: actions/cache/restore@v4
         id: cpp-wasm-cache
         with:
           path: ./wasm
@@ -184,9 +184,6 @@ jobs:
       - name: "Upload all WASM files to the cluster"
         run: ./bin/inv_wrapper.sh wasm.upload
       - name: "Run MPI kernels"
-        # TODO: MPI kernels still not run in WAMR because there's a seg. fault
-        # with the `exit` syscall
-        if: "contains(env.FAASM_WASM_VM, 'wavm')"
         timeout-minutes: 2
         run: |
           # The global MPI kernel stopped working when enabling SIMD

FAASM_VERSION

@@ -1,47 +1,6 @@
 version: "3"
 
 services:
-  redis:
-    image: faasm.azurecr.io/redis:${EXAMPLES_RUN_VERSION}
-
-  planner:
-    image: faasm.azurecr.io/planner:0.11.0
-    environment:
-      - LOG_LEVEL=info
-      - PLANNER_PORT=8081
-
-  minio:
-    image: faasm.azurecr.io/minio:${EXAMPLES_RUN_VERSION}
-    environment:
-      MINIO_ROOT_USER: minio
-      MINIO_ROOT_PASSWORD: minio123
-    volumes:
-      - ./dev/minio/data:/data/minio
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 5s
-      timeout: 3s
-      retries: 3
-
-  run:
-    depends_on:
-      - redis
-      - minio
-    image: faasm.azurecr.io/examples-run:${EXAMPLES_RUN_VERSION}
-    working_dir: /usr/local/code/faasm
-    stdin_open: true
-    tty: true
-    volumes:
-      - ./:/code/examples
-      - ./dev/faasm-local/:/usr/local/faasm
-    environment:
-      - DEFAULT_MPI_WORLD_SIZE=4
-      - LOG_LEVEL=info
-      - LD_LIBRARY_PATH=/build/faasm/third-party/lib:/usr/local/lib
-      - REDIS_QUEUE_HOST=redis
-      - REDIS_STATE_HOST=redis
-      - FAASM_WASM_VM=${FAASM_WASM_VM:-wavm}
-
   build:
     image: faasm.azurecr.io/examples-build:${EXAMPLES_BUILD_VERSION}
     working_dir: /code/examples

docker-compose.yml

@@ -1,4 +1,5 @@
 black>=22.3.0
-faasmctl>=0.23.0
+faasmctl>=0.28.0
 flake8>=4.0.1
 invoke>=1.7.1
+PyYAML>=6.0.1

docker/run.dockerfile

@@ -8,7 +8,6 @@
     DEV_FAASM_LOCAL,
     EXAMPLES_BUILD_IMAGE_NAME,
     PROJ_ROOT,
-    get_faasm_version,
     get_version,
 )
 
@@ -40,12 +39,7 @@ def cli(ctx, service, clean=False):
         run("docker rm -f {}".format(tmp_ctr_name), shell=True, check=True)
 
     build_env = environ.copy()
-    build_env.update(
-        {
-            "EXAMPLES_RUN_VERSION": get_faasm_version(),
-            "EXAMPLES_BUILD_VERSION": get_version(),
-        }
-    )
+    build_env.update({"EXAMPLES_BUILD_VERSION": get_version()})
     docker_cmd = "docker compose up -d --no-recreate {}".format(service)
     run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT, env=build_env)
 
@@ -56,12 +50,7 @@ def cli(ctx, service, clean=False):
 @task()
 def stop(ctx):
     build_env = environ.copy()
-    build_env.update(
-        {
-            "EXAMPLES_RUN_VERSION": get_faasm_version(),
-            "EXAMPLES_BUILD_VERSION": get_version(),
-        }
-    )
+    build_env.update({"EXAMPLES_BUILD_VERSION": get_version()})
     run(
         "docker compose down",
         shell=True,

requirements.txt

@@ -3,8 +3,7 @@
 from faasmtools.env import get_version as get_cpp_version
 from os import environ
 from os.path import dirname, abspath, join
-from re import search as re_search
-from subprocess import run
+from yaml import safe_load
 
 PROJ_ROOT = dirname(dirname(abspath(__file__)))
 EXAMPLES_IN_DOCKER_ROOT = "/code/examples"
@@ -69,55 +68,22 @@ def get_python_version():
 
 def get_faasm_version():
     """
-    Get the version of the python submodule
+    Get the version of the Faasm dependency
     """
-    ver_file = join(PROJ_ROOT, "FAASM_VERSION")
-
-    with open(ver_file, "r") as fh:
-        version = fh.read()
-
-    version = version.strip()
-    return version
-
-
-def get_faabric_version(old_faasm_ver, new_faasm_ver):
-    """
-    Get the faabric version by `wget`-ing the FAASM version file for the tagged
-    Faasm version
-    """
-
-    def do_get_ver(faasm_ver):
-        tmp_file = "/tmp/faabric_version"
-        wget_cmd = [
-            "wget",
-            "-O {}".format(tmp_file),
-            "https://raw.githubusercontent.com/faasm/faasm/v{}/.env".format(
-                faasm_ver
-            ),
-        ]
-        wget_cmd = " ".join(wget_cmd)
-        out = run(wget_cmd, shell=True, capture_output=True)
-        assert out.returncode == 0
-
-        with open(tmp_file, "r") as fh:
-            ver = re_search(r"planner:([0-9\.]*)", fh.read()).groups(1)[0]
-
-        return ver
-
-    return do_get_ver(old_faasm_ver), do_get_ver(new_faasm_ver)
+    yaml_path = join(PROJ_ROOT, ".github", "workflows", "tests.yml")
+    return safe_load(open(yaml_path, "r"))["jobs"]["run-examples-faasmctl"][
+        "env"
+    ]["FAASM_VERSION"]
 
 
-def get_version(name="build"):
+def get_version():
     """
     Get version for the examples repository
 
     The version depends only on the version of the different cross-compilation
-    toolchains we use, in this case CPP and Python, and Faasm
+    toolchains we use, in this case CPP and Python
     """
-    if name == "build":
-        return "{}_{}".format(get_cpp_version(), get_python_version())
-    if name == "run":
-        return "{}".format(get_faasm_version())
+    return "{}_{}".format(get_cpp_version(), get_python_version())
 
 
 def in_docker():

tasks/cli.py

@@ -3,16 +3,15 @@
 from tasks.env import (
     EXAMPLES_BUILD_IMAGE_NAME,
     PROJ_ROOT,
-    get_faabric_version,
     get_faasm_version,
     get_version,
 )
 
 VERSIONED_FILES = {
-    "faasm": ["FAASM_VERSION", ".github/workflows/tests.yml"],
-    "faabric": ["docker-compose.yml", ".github/workflows/tests.yml"],
+    "faasm": [".github/workflows/tests.yml"],
     "cpp": [".github/workflows/tests.yml"],
     "python": [".github/workflows/tests.yml"],
+    "faasmctl": ["requirements.txt"],
 }
 
 
@@ -41,9 +40,9 @@ def tag(ctx, force=False):
 @task
 def bump(ctx, submodule, ver=None):
     """
-    Bump a submodule's dependency: `faasm`, `cpp`, or `python`
+    Bump a submodule's dependency: `faasm`, `cpp`, `python`, or `faasmctl`
     """
-    allowed_submodules = ["faasm", "cpp", "python"]
+    allowed_submodules = ["faasm", "cpp", "python", "faasmctl"]
     if submodule not in allowed_submodules:
         print("Unrecognised submodule: {}".format(allowed_submodules))
         print("Submodule must be one in: {}".format(allowed_submodules))
@@ -61,12 +60,6 @@ def bump(ctx, submodule, ver=None):
             sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f)
             run(sed_cmd, shell=True, check=True)
 
-        # Also update the planner version accordingly
-        old_fabric_ver, new_faabric_ver = get_faabric_version(old_ver, new_ver)
-        for f in VERSIONED_FILES["faabric"]:
-            sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f)
-            run(sed_cmd, shell=True, check=True)
-
     else:
         new_ver = get_version("build")
         grep_cmd = "grep '{}' .github/workflows/tests.yml".format(
@@ -83,3 +76,8 @@ def bump(ctx, submodule, ver=None):
         for f in VERSIONED_FILES[submodule]:
             sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f)
             run(sed_cmd, shell=True, check=True)
+
+
+@task
+def foo(ctx):
+    print(get_faasm_version())