Prepares action to add arm32 device (antmicro#42)

Adds architecture class to store important information about arch
Adds board-specific tasks
Changes kernel package definition and add support for vmlinux image type

action/common.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from dataclasses import dataclass
 from urllib.parse import urlparse
+from typing import Dict
 
 import os
 import re
@@ -22,6 +24,24 @@
 import pexpect as px
 
 
+@dataclass
+class Architecture:
+    python_name: str
+    docker_name: str
+    default_board: str
+    network_available: bool
+
+
+archs: Dict[str, Architecture] = {
+    "riscv64": Architecture(
+        python_name="riscv64",
+        docker_name="riscv64",
+        default_board="hifive_unleashed",
+        network_available=True,
+    ),
+}
+
+
 class FilteredStdout(object):
     """
     Stdout wrapper which replaces found pattern with 'replace' string.

action/dependencies.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from common import run_cmd, error
+from common import run_cmd, error, archs
 from images import shared_directories_action, shared_directories_actions
 
 from typing import Dict
@@ -44,7 +44,7 @@ def get_package(child: px.spawn, arch: str, package_name: str) -> list[str]:
     """
 
     child.sendline('')
-    run_cmd(child, "(venv-dir) #", f"pip download {package_name} --platform=linux_{arch} --no-deps --progress-bar off --disable-pip-version-check")
+    run_cmd(child, "(venv-dir) #", f"pip download {package_name} --platform=linux_{archs[arch].python_name} --no-deps --progress-bar off --disable-pip-version-check")
     child.expect_exact('(venv-dir) #')
 
     # Removes strange ASCII control codes that appear during some 'pip download' runs.

action/dispatcher.py

@@ -35,7 +35,7 @@ class CommandDispatcher:
     tasks: Dict[str, Task] = {}
     default_vars: Dict[str, str] = {}
 
-    def __init__(self, global_vars: Dict[str, str], override_vars: Dict[str, Dict[str, str]]) -> None:
+    def __init__(self, board: str, global_vars: Dict[str, str], override_vars: Dict[str, Dict[str, str]]) -> None:
         """
         Parameters
         ----------
@@ -62,12 +62,12 @@ def __init__(self, global_vars: Dict[str, str], override_vars: Dict[str, Dict[st
         self.shells = {name: self.add_shell(name, *term) for (name, term) in init_shells.items()}
 
         self._add_default_vars(global_vars)
-        self._load_tasks(override_vars)
+        self._load_tasks(board, override_vars)
 
     def _add_default_vars(self, vars: Dict[str, str]) -> None:
         self.default_vars.update(vars)
 
-    def _load_tasks(self, override_vars: Dict[str, Dict[str, str]]) -> None:
+    def _load_tasks(self, board: str, override_vars: Dict[str, Dict[str, str]]) -> None:
         """
         Loads Tasks from YAML files stored in 'action/tasks' and 'action/user_tasks' directories and adds them to the `tasks` dict
 
@@ -76,15 +76,16 @@ def _load_tasks(self, override_vars: Dict[str, Dict[str, str]]) -> None:
         override_vars: dictionary that stores different dictionaries for each task to override existing variables there
         """
 
-        for directory in ["tasks", "user_tasks"]:
-            for path, _, files in os.walk(f"action/{directory}"):
-                for f in files:
-                    fp = os.path.join(path, f)
-                    if fp.endswith((".yml", ".yaml")):
-                        with open(fp) as task_file:
-                            task = Task.load_from_yaml(task_file.read())
-                            task.apply_vars(self.default_vars, override_vars.get(task.name, {}))
-                            self.add_task(task)
+        for directory in ["action/tasks", f"action/device/{board}/tasks", "action/user_tasks"]:
+            if os.path.exists(directory):
+                for path, _, files in os.walk(directory):
+                    for f in files:
+                        fp = os.path.join(path, f)
+                        if fp.endswith((".yml", ".yaml")):
+                            with open(fp) as task_file:
+                                task = Task.load_from_yaml(task_file.read())
+                                task.apply_vars(self.default_vars, override_vars.get(task.name, {}))
+                                self.add_task(task)
 
     def _sort_tasks(self) -> None:
         """

action/images.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from common import run_cmd, get_file, error
+from common import get_file, error, archs
 
 from subprocess import run, DEVNULL, CalledProcessError
 from dataclasses import dataclass
@@ -25,7 +25,6 @@
 import tarfile
 import requests
 import dockersave
-import pexpect as px
 
 
 @dataclass
@@ -102,22 +101,8 @@ def prepare_kernel_and_initramfs(kernel: str):
     with tarfile.open("kernel.tar.xz") as tar:
         tar.extractall("images")
 
-    child = px.spawn(f'sh -c "cd {os.getcwd()};exec /bin/sh"', encoding="utf-8", timeout=10)
-
-    try:
-        child.expect_exact('#')
-        child.sendline('')
-
-        run_cmd(child, "#", "mkdir -p images/initramfs")
-        run_cmd(child, "#", "cd images/initramfs && cpio -iv < ../rootfs.cpio")
-        run_cmd(child, "#", f"cd {os.getcwd()}")
-        run_cmd(child, "#", "cp images/initramfs/boot/Image images")
-        run_cmd(child, "#", "cp images/initramfs/boot/*.dtb images")
-        run_cmd(child, "#", "rm -rf images/initramfs")
-
-        child.expect_exact('#')
-    except px.TIMEOUT:
-        error("Timeout!")
+    if not os.path.exists("images/Image") and not os.path.exists("images/vmlinux"):
+        error("Kernel not found! Action expects Image or vmlinux file.")
 
 
 def burn_rootfs_image(
@@ -157,7 +142,7 @@ def burn_rootfs_image(
             image_proto = dockersave.Image(
                     image=f"{library}/{image}",
                     tag=tag,
-                    arch=arch
+                    arch=archs[arch].docker_name
             )
         except StopIteration:
             print(f"This package is not available for the selected architecture: {arch}!")

action/run-in-renode.py

@@ -12,15 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from common import get_file, error
 from command import Task
+from common import get_file, error, archs
 from devices import add_devices
 from dependencies import add_repos, add_packages
 from images import prepare_shared_directories, prepare_kernel_and_initramfs, burn_rootfs_image
 from dispatcher import CommandDispatcher
 
 from datetime import datetime
-from typing import Dict
 
 import sys
 import json
@@ -31,11 +30,6 @@
 DEFAULT_KERNEL_PATH = "https://github.com/{}/releases/download/{}/kernel-{}-{}.tar.xz"
 
 
-default_boards: Dict[str, str] = {
-    "riscv64": "hifive_unleashed"
-}
-
-
 def configure_board(arch: str, board: str, resc: str, repl: str):
     """
     Set the appropriate board resc and repl
@@ -52,11 +46,11 @@ def configure_board(arch: str, board: str, resc: str, repl: str):
         custom repl: URL or path
     """
 
-    if arch not in default_boards:
+    if arch not in archs:
         error("Architecture not supportted!")
 
     if board == "default":
-        board = default_boards[arch]
+        board = archs[arch].default_board
 
     if board == "custom" and (resc == "default" or repl == "default"):
         error("You have to provide resc and repl for custom board")
@@ -137,15 +131,15 @@ def test_task(test_task_str: str):
     for it, custom_task in enumerate(args.get("tasks", "").splitlines()):
         get_file(custom_task, f"action/user_tasks/task{it}.yml")
 
-    dispatcher = CommandDispatcher({
+    dispatcher = CommandDispatcher(board, {
         "NOW": str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
         "BOARD": board
     }, optional_tasks)
 
     for task in optional_tasks:
         dispatcher.enable_task(task, True)
 
-    if args.get("network", "true") != "true":
+    if args.get("network", "true") != "true" or not archs[arch].network_available:
         for i in ["host", "renode", "target"]:
             dispatcher.enable_task(f"{i}_network", False)
 

docs/Kernel.md

@@ -27,13 +27,13 @@ There is also a Busybox with essential commands.
 
 Your custom `kernel` package should include:
 
-* `fw_payload.elf`: the bootloader and firmware for the HiFive Unleashed platform
-* `rootfs.cpio`: a packed `initramfs` compiled for the selected architecture (`riscv64`)
+* The file `vmlinux` or `Image`: the compiled kernel for the selected architecture with the bootloader and firmware if vmlinux is provided
+* `fw_payload.elf`: (for `Image` kernel) the bootloader and firmware for the board
+* `rootfs.cpio`: a packed `initramfs` compiled for the selected architecture
+* `.dtb` file: the device tree binary file for the specified board with the `.dtb` file extension.
 
 `rootfs.cpio` should contain:
 
-* in the `/boot` directory, the file `Image` that should be the compiled kernel for the selected architecture.
-* in the `/boot` directory, the device tree binary file for the specified board with the `.dtb` file extension.
 * `/init`, an executable script that redirects output to `ttyS0` device and starts an interactive shell session.
 * basic programs like: `sh`, `mount`, `chroot`, `dmesg`, `date`.
 * If you want to use networking, you should provide the `ip` command and network stack.
@@ -63,7 +63,7 @@ make -j$(nproc)
 
 This will take some time.
 
-Eventually, you should have some files in the `buildroot/output/images` directory. Create the new `tar.xz` archive with the files: `rootfs.cpio` and `fw_payload.elf`. The resulting archive is ready to use with the action.
+Eventually, you should have some files in the `buildroot/output/images` directory. Create the new `tar.xz` archive with [required files](#required-image-components). The resulting archive is ready to use with the action.
 
 ### Use your kernel
 

scripts/compile-kernel.sh

@@ -6,7 +6,7 @@ BOARD="$2"
 
 # This script has 2 parameters: Processor architecture and board name
 
-mkdir -p images
+mkdir -p images tmp
 git clone --branch $BUILDROOT_VERSION https://github.com/buildroot/buildroot buildroot
 
 cd buildroot
@@ -18,6 +18,24 @@ cd ..
 
 # Information about needed files in docs/Kernel.md
 
-cp buildroot/output/images/{fw_payload.elf,rootfs.cpio} .
-tar cJvf images/kernel-$ARCH-$BOARD.tar.xz ./{fw_payload.elf,rootfs.cpio}
-rm -rf ./{fw_payload.elf,rootfs.cpio} buildroot
+if [ -f buildroot/output/images/fw_payload.elf ]; then
+    cp buildroot/output/images/fw_payload.elf tmp
+fi
+
+if [ -f buildroot/output/images/Image ]; then
+    cp buildroot/output/images/Image tmp
+elif [ -f buildroot/output/images/vmlinux ]; then
+    cp buildroot/output/images/vmlinux tmp
+else
+    echo "Kernel not found!"
+    exit 1
+fi
+
+cp buildroot/output/images/rootfs.cpio tmp
+cp buildroot/output/images/*.dtb tmp
+
+cd tmp
+tar cJvf ../images/kernel-$ARCH-$BOARD.tar.xz ./*
+cd ..
+
+rm -rf tmp buildroot