version 0.1

.github/workflows/tests.yml

@@ -0,0 +1,33 @@
+name: unit tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        python-version: [3.9.2]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install poetry
+        poetry install --with test
+    - name: Run tests
+      run: |
+        poetry run pytest

.gitignore

@@ -0,0 +1,17 @@
+# Python bytecode
+__pycache__/
+.py[cod]
+
+# Environments
+venv
+
+# Coverage reports
+.coverage
+*.xml
+
+# mypy
+.mypy_cache/
+
+# Editors
+*~
+.vscode

README.md

@@ -0,0 +1,81 @@
+[![Python package](https://github.com/MPI-IS/nightskycam-serialization/actions/workflows/tests.yml/badge.svg)](https://github.com/MPI-IS/nightskycam-serialization/actions/workflows/tests.yml)
+[![PyPI version](https://img.shields.io/pypi/v/nightskycam-serialization.svg)](https://pypi.org/project/nightskycam-serialization/)
+
+
+> 🚧 **Under Construction**  
+> This project is currently under development. Please check back later for updates.
+
+
+# Nightskycam Serialization
+
+## About
+
+This is is a support package for:
+
+- [nightskycam](https://gitlab.is.tue.mpg.de/allsky/nightskycam)
+- [nightskycam-server](https://gitlab.is.tue.mpg.de/allsky/nightskycam-server)
+
+These two packages do not depend on each other, yet the services they spawn will communicate with each other in the form of
+serialized messages (sent via websockets). This requires the code in these two packages to follow the same convention on how
+messages are serialized and deserialized. To enforce this, these two packages are dependant on the nightskycam-serialization
+package and use its API for serializing / deserializing messages.
+
+## Getting Started as a User (using `pip`)
+
+Dependency management with `pip` is easier to set up than with `poetry`, but the optional dependency-groups are not installable with `pip`.
+
+* Create and activate a new Python virtual environment:
+  ```bash
+  python3 -m venv --copies venv
+  source venv/bin/activate
+  ```
+* Update `pip` and build package:
+  ```bash
+  pip install -U pip  # optional but always advised
+  pip install .       # -e option for editable mode
+  ```
+
+## Getting Started as a Developer (using `poetry`)
+
+Dependency management with `poetry` is required for the installation of the optional dependency-groups.
+
+* Install [poetry](https://python-poetry.org/docs/).
+* Install dependencies for package
+  (also automatically creates project's virtual environment):
+  ```bash
+  poetry install
+  ```
+* Install `dev` dependency group:
+  ```bash
+  poetry install --with dev
+  ```
+* Activate project's virtual environment:
+  ```bash
+  poetry shell
+  ```
+
+## Tests (only possible for setup with `poetry`, not with `pip`)
+
+
+To install `test` dependency group:
+```bash
+poetry install --with test
+```
+
+To run the tests:
+```bash
+python -m pytest tests
+```
+
+To extract coverage data:
+* Get code coverage by measuring how much of the code is executed when running the tests:
+  ```bash
+  coverage run -m pytest tests
+  ```
+* View coverage results:
+  ```bash
+  # Option 1: simple report in terminal.
+  coverage report
+  # Option 2: nicer HTML report.
+  coverage html  # Open resulting 'htmlcov/index.html' in browser.
+  ```

mypy.ini

@@ -0,0 +1,7 @@
+[mypy]
+
+[mypy-tomli_w.*]
+ignore_missing_imports = True
+
+[mypy-pytest.*]
+ignore_missing_imports = True

nightskycam_serialization/command.py

@@ -0,0 +1,124 @@
+from dataclasses import asdict, dataclass
+from typing import Dict, Optional, Tuple, Union
+
+from .serialize import ImproperMessage, deserialize, serialize
+
+
+def serialize_command(
+    command_id: int, command: str, token: Optional[str] = None
+) -> str:
+    """
+    Serialize a command to be executed on the system
+
+    Counterpart, see: [deserialize_command]().
+
+    Arguments
+      command_id: unique identifier of the command
+      command: the command to execute
+      token: see [serialize.serialize]()
+
+    Returns
+      The serialized message
+    """
+    d: Dict[str, Union[int, str]] = {
+        "command_id": command_id,
+        "command": command,
+    }
+    return serialize(d, token=token)
+
+
+def deserialize_command(message: str, token: Optional[str] = None) -> Tuple[int, str]:
+    """
+    Deserialize a message sent by the server into a command to
+    be executed by the system.
+
+    Counterpart, see [serialize_command]()
+
+    Arguments
+      message: the serialized message
+      token: see [serialize.deserialize]()
+
+    Returns
+      command_id: unique identifier of the command
+      command: the command to execute
+
+
+    Raises
+      [improper_message.ImproperMessage]() if message is not properly
+      formated.
+    """
+
+    data = deserialize(
+        message,
+        required_keys=("command_id", "command"),
+        token=token,
+    )
+    try:
+        command_id = int(data["command_id"])
+    except ValueError:
+        raise ImproperMessage(
+            "value for key 'command_id' should be an int, "
+            f"but received value {data['command_id']} of type {type(data['command_id'])} instead."
+        )
+
+    return command_id, str(data["command"])
+
+
+@dataclass
+class CommandResult:
+    """
+    Summary of the "output" of a command executed on the system.
+    """
+
+    command_id: int
+    command: str
+    stdout: str = ""
+    stderr: str = ""
+    exit_code: str = ""
+    error: str = ""
+
+
+def serialize_command_result(result: CommandResult, token: Optional[str] = None) -> str:
+    """
+    Serialize a command output to a string.
+
+    Counterpart method: [deserialize_command_result]().
+
+    Arguments
+      result: output of the command
+      token: see [serialize.serialize]()
+
+    Returns
+      serialized result
+    """
+    data = asdict(result)
+    return serialize(data, token=token)
+
+
+def deserialize_command_result(
+    message: str, token: Optional[str] = None
+) -> CommandResult:
+    """
+    Deserialize a message into an instance of CommandResult
+
+    Counterpart method: [serialize_command_result]().
+
+    Arguments:
+      message: the serialized message
+      token: see [serialize.deserialize]()
+
+    Returns
+      The corresponding instance of CommandResult
+
+    Raises
+      [improper_message.ImproperMessage]() if message is not properly formatted.
+    """
+
+    data = deserialize(message, required_keys=tuple(), token=token)
+    try:
+        return CommandResult(**data)
+    except TypeError as te:
+        raise ImproperMessage(
+            "failed to deserialize the message into an instance "
+            f"of CommandResult: {te}"
+        )

nightskycam_serialization/config.py

@@ -0,0 +1,65 @@
+from typing import Any, Dict, Optional, Tuple
+
+from .fix import deserialize_fix_dict, serialize_fix_dict
+from .serialize import ImproperMessage, deserialize, serialize
+
+_None = "__None__"
+
+
+def serialize_config_update(
+    runner_name: str, config: Dict[str, Any], token: Optional[str] = None
+) -> str:
+    """
+    For server to request runners of remote system to reconfigure
+    themselves.
+
+    Counterpart: [deserialize_config_update]()
+
+    Arguments
+      runner_name: name of the runner that needs reconfiguring
+      config: the new configuration to apply.
+      token: see [serialize.serialize]()
+
+    Returns
+      corresponding serialized message
+    """
+
+    d = {"runner_name": runner_name, "config": serialize_fix_dict(config)}
+    return serialize(d, token=token)
+
+
+def _deserialize_fix(value: Any) -> Any:
+    # Casting back "__None__" to None.
+    if isinstance(value, str) and value == "__None__":
+        return None
+    return value
+
+
+def _deserialize_fix_config(config: Dict[str, Any]) -> Dict[str, Any]:
+    return {k: _deserialize_fix(v) for k, v in config.items()}
+
+
+def deserialize_config_update(
+    message: str, token: Optional[str] = None
+) -> Tuple[str, Dict[str, Any]]:
+    """
+    For system receiving configuration update request for a runner.
+
+    Counterpart: [serialize_config_update]()
+
+    Arguments
+      message: the serialized request
+      token: see [serialize.deserialize]()
+
+    Returns
+      Tuple:
+      - the name of the runner that should update
+      - the configuration
+    """
+
+    data = deserialize(message, required_keys=("runner_name", "config"), token=token)
+    if not isinstance(data["config"], dict):
+        raise ImproperMessage(
+            f"configuration expected to be a dictionary, but got {data['config']} of type {type(data['config'])} instead."
+        )
+    return (str(data["runner_name"]), deserialize_fix_dict(data["config"]))

nightskycam_serialization/fix.py

@@ -0,0 +1,62 @@
+from pathlib import Path
+from typing import Any, Dict, TypeVar
+
+from nightskyrunner.status import StatusDict
+
+_None = "__None__"
+
+
+SerialDict = TypeVar("SerialDict", Dict[str, Any], StatusDict)
+
+
+def serialize_fix(value: Any) -> Any:
+    """
+    Sometimes python dictionary can not be serialized into
+    toml. Here fixing the commonly encountered issue:
+
+    - pathlib.Path (cast to string)
+    - None (cast to the string "__None__")
+
+    See: [deserialize_fix]()
+    """
+
+    if value is None:
+        return _None
+    if isinstance(value, Path):
+        return str(value)
+    return value
+
+
+def serialize_fix_dict(config: SerialDict) -> SerialDict:
+    """
+    Applying [serialize_fix] to all values of the dictionary
+
+    See: [deserialize_fix_dict]()
+    """
+    instance = type(config)()
+    for k, v in config.items():
+        instance[k] = serialize_fix(v)  # type: ignore
+    return instance
+
+
+def deserialize_fix(value: Any) -> Any:
+    """
+    Casting the string "__None__" to None.
+
+    See: [serialize_fix]()
+    """
+
+    # Casting back "__None__" to None.
+    if isinstance(value, str) and value == "__None__":
+        return None
+    return value
+
+
+def deserialize_fix_dict(config: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Applying [deserialize_fix]() to all values of the dictionary.
+
+    See: [serialize_fix_dict]().
+    """
+
+    return {k: deserialize_fix(v) for k, v in config.items()}