Removed requirement to install terminal_sync; updated Dockerfile

- Updated Dockerfile to export requirements.txt (to ensure platform compatibility) and install using pip rather than PDM (to reduce runtime complexity)
- Increased minimum Python version to 3.10 due to use of PEP-604 style type hinting syntax
- Reduced default timeout values as the previous ones were painfully long if something failed and would negatively impact user experience
- Minor updates to the README
- Updated CHANGELOG

Author: Dan Breakiron

CHANGELOG.md

@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [v0.2.0] - 2023-04-26
+
+### Added
+
+- Added a new feature to save failed (or optionally all) logs locally (as JSON files)
+- Created a script / library to export the JSON files to a GhostWriter CSV
+  - Configured the server to automatically export a CSV file on shutdown (though stopping a Docker container doesn't seem to trigger it)
+
+### Fixed
+
+- Fixed several syntax errors in the Bash install / hook script that prevented successful installation
+
+### Changed
+
+- Updated Dockerfile to export requirements.txt (to ensure platform compatibility) and install using pip rather than PDM (to reduce runtime complexity)
+- Increased minimum Python version to 3.10 due to use of PEP-604 style type hinting syntax
+- Reduced default timeout values as the previous ones were painfully long if something failed and would negatively impact user experience
+
 ## [v0.1.0] - 2023-04-21
 
 Initial release

Dockerfile

@@ -1,34 +1,43 @@
-FROM python:3.11
+# Install PDM and use it to export requirements.txt to ensure platform compatibility
+FROM python:3.11-slim as requirements-stage
 
-ENV LANG=C.UTF-8
+WORKDIR /tmp
+
+RUN pip install pdm
+
+# Copy the pyproject.toml and pdm.lock files to the /tmp directory
+# Because it uses ./pdm.lock* (ending with a *), it won't crash if that file is not available
+COPY ./pyproject.toml ./pdm.lock* /tmp/
+
+# Generate requirements.txt
+RUN pdm export --prod --without-hashes -o requirements.txt
+
+# Build the final image
+FROM python:3.11-slim
+
+# ENV LANG=C.UTF-8  # Defined in the base image
 ENV LC_ALL=C.UTF-8
 ENV LC_TIME=C.UTF-8
 ENV LISTEN_PORT=8000
 # Enable Python optimizations
 ENV PYTHONOPTIMIZE=1
 
-# Create a directory where cached log entries will be stored
-# VOLUME /app/log_archive
-
 WORKDIR /app
 
-# COPY requirements.txt requirements.txt
-
-# RUN pip install --no-cache-dir --upgrade -r requirements.txt
-
-# COPY ./src/terminal_sync .
-
-RUN pip install pdm
+# Copy the requirements.txt file from the previous stage
+COPY --from=requirements-stage /tmp/requirements.txt /app/requirements.txt
 
-COPY . .
+# Upgrade pip and install / upgrade required packages
+RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir --upgrade -r requirements.txt
 
-RUN pdm install --prod
+# Copy the application code into the container
+COPY ./src/terminal_sync ./terminal_sync
 
 EXPOSE $LISTEN_PORT/tcp
 
 # FastAPI has a built-in /docs endpoint that we can use to check whether the server is running properly
 HEALTHCHECK CMD curl --fail http://localhost:$LISTEN_PORT/docs || exit 1
 
+# Run terminal_sync as a module
 # IMPORTANT: This must listen on 0.0.0.0 or else the application will not be accessible outside of the container
-CMD pdm run uvicorn terminal_sync.api:app --host 0.0.0.0 --port $LISTEN_PORT
-# CMD ["pdm", "run", "uvicorn", "terminal_sync.api:app", "--host", "0.0.0.0", "--port", $LISTEN_PORT]
+CMD python3 -m terminal_sync --host 0.0.0.0 --port $LISTEN_PORT

README.md

@@ -74,12 +74,12 @@ For the purpose of this documentation, the terms "API key" and "token" are used
     - Bash
 
         1. Edit the **Configuration Settings** section at the top of `terminal_sync.sh`
-        2. Run `chmod +x ./terminal_sync.sh && ./terminal_sync.sh`
+        2. Run `source ./terminal_sync.sh`
             - **Note**: This must be done in each new bash session
             - You will be prompted to install any missing dependencies
             - On first run, you will also be prompted whether you want to install the hooks permanently
 
-        - If you want to install the hooks later, just append `source <PATH_TO>/terminal_sync.sh` to your `~/.bashrc` file.
+        - If you want to install the hooks later, just append `source '<PATH_TO>/terminal_sync.sh'` to your `~/.bashrc` file.
 
     - PowerShell
 
@@ -139,7 +139,7 @@ To permanently disable terminal_sync, delete or comment out the line in `~/.bash
 
 Both the Bash script and PowerShell module contain a configuration setting that allows you to set the default console output. If you want to (temporarily) change this for a session, use one of the following commands.
 
-In Bash, run `set_termsync_versbosity`; this will print a list of display settings and prompt you for a number. Simply enter the number that matches your preferred setting.
+In Bash, run `Set-TermSyncVersbosity`; this will print a list of display settings and prompt you for a number. Simply enter the number that matches your preferred setting.
 
 In PowerShell, type `Set-TermSyncVerbosity`, followed by a space, then press **TAB** to cycle through the available options. Press **Enter** to select your preferred setting.
 
@@ -165,6 +165,8 @@ Development and testing were performed using the following configurations. If yo
   - Docker Compose v2.17.2
   - PowerShell 5.1.22621.963
   - Python 3.11
+- Debian 11 (terminal_sync server and Bash hooks)
+  - Python 3.11.3
 - Debian 11 (Bash hooks)
   - Python 3.9.2
 - GhostWriter running over HTTPS

pdm.lock

@@ -4,12 +4,12 @@ build-backend = "pdm.backend"
 
 [project]
 name = "terminal_sync"
-version = "0.1.0"
+version = "0.2.0"
 description = "A standalone tool for logging shell commands to GhostWriter automatically"
 authors = [{ name = "breakid" }]
 license = { text = "GPL-3.0-or-later" }
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.10" # The type hinting used in this project follows PEP 604, which was implemented in 3.10
 dependencies = [
   "aiohttp>=3.8.4",
   "fastapi>=0.95.0",

pyproject.toml

@@ -1 +1,5 @@
 """Marks the current directory as a Python package"""
+
+# Note: Version is also defined in pyproject.toml. Ideally, to ensure they don't get out of sync, version would
+# only be defined once; however, the project one is only accessible if terminal_sync is installed as a package
+__version__ = "0.2.0"

src/terminal_sync/__init__.py

@@ -1,5 +1,14 @@
 """Executes the terminal_sync server when the module is run"""
+
+# Standard Libraries
+from argparse import ArgumentParser
+
 # Internal Libraries
 from terminal_sync.api import run
 
-run()
+parser = ArgumentParser()
+parser.add_argument("--host", dest="host", default="127.0.0.1", help="The host address where the server will bind")
+parser.add_argument("--port", dest="port", default=8000, type=int, help="The host port where the server will bind")
+args = parser.parse_args()
+
+run(host=args.host, port=args.port)

src/terminal_sync/__main__.py

@@ -271,7 +271,7 @@ async def post_exec(msg: Message) -> str:
 async def app_shutdown() -> None:
     """Export a GhostWriter CSV file on server shutdown"""
     try:
-        csv_filepath: Path = export_csv(Path(config.termsync_json_log_dir))
+        csv_filepath: Path = export_csv(Path(config.termsync_json_log_dir), Path(config.termsync_json_log_dir))
         logger.info(f"Exported cached logs to: {csv_filepath}")
     except Exception as e:
         logger.exception(e)

src/terminal_sync/api.py

@@ -51,7 +51,7 @@ class Config:
     termsync_listen_host: str = "127.0.0.1"
     termsync_listen_port: int = 8000
     termsync_save_all_local: bool = False
-    termsync_timeout_seconds: int = 10
+    termsync_timeout_seconds: int = 3
 
     # TODO: Automatically add keywords "exported" by registered command parsers
     # TODO: This list should only contain keywords that don't have an associated command parser

src/terminal_sync/config.py

@@ -45,4 +45,4 @@ termsync_keywords:
 #termsync_save_all_local: no
 
 # The number of seconds the server will wait for a response from GhostWriter
-#termsync_timeout_seconds: 10
+#termsync_timeout_seconds: 3

src/terminal_sync/config_template.yaml

@@ -2,10 +2,7 @@
 
 # Standard Libraries
 import logging
-
-# from asyncio.exceptions import TimeoutError
 from collections.abc import Callable
-from importlib import metadata
 
 # Third-party Libraries
 import aiohttp
@@ -15,6 +12,7 @@
 from gql.transport.aiohttp import AIOHTTPTransport
 
 # Internal Libraries
+from terminal_sync import __version__ as termsync_version
 from terminal_sync.log_entry import Entry
 
 logger = logging.getLogger("terminal_sync")
@@ -113,7 +111,7 @@ def __init__(self, url: str, graphql_api_key: str = "", rest_api_key: str = "",
         self.base_url: str = url.rstrip("/")
 
         self.headers: dict[str, str] = {
-            "User-Agent": f"terminal_sync/{metadata.version('terminal_sync')}",
+            "User-Agent": f"terminal_sync/{termsync_version}",
             "Authorization": f"Bearer {graphql_api_key}" if graphql_api_key else f"Api-Key {rest_api_key}",
             "Content-Type": "application/json",
         }
@@ -123,9 +121,15 @@ def __init__(self, url: str, graphql_api_key: str = "", rest_api_key: str = "",
         self.update_log: Callable = self._update_entry_graphql
 
         if graphql_api_key:
+            logger.info("Using the GraphQL API")
             url = f"{self.base_url}/v1/graphql"
-            self._transport: AIOHTTPTransport = AIOHTTPTransport(url=url, timeout=timeout_seconds, headers=self.headers)
+            # WORKAROUND: When running Docker on a Windows host, the application will always hang waiting for the SSL
+            # connection to terminate. The ssl_close_timeout is therefore set to 0 to avoid a negative user experience
+            self._transport: AIOHTTPTransport = AIOHTTPTransport(
+                url=url, headers=self.headers, ssl_close_timeout=0, timeout=timeout_seconds
+            )
         else:
+            logger.info("Using the REST API")
             # Important: If you leave off the trailing "/" on oplog/api/entries/ then this POST will return "200 OK"
             # without actually doing anything
             self.rest_url: str = f"{self.base_url}/oplog/api/entries/"

src/terminal_sync/ghostwriter.py

@@ -20,7 +20,7 @@ $global:TermSyncVerbosity = 4
 $global:TermSyncServer = "127.0.0.1:8000"
 
 # The number of seconds the client will wait for a response from the terminal_sync server
-$global:TimeoutSec = 10
+$global:TimeoutSec = 4
 
 
 # =============================================================================

terminal_sync.psm1

@@ -14,7 +14,7 @@ TERMSYNC_LOGGING=1
 TERMSYNC_VERBOSITY=4
 
 # The number of seconds the client will wait for a response from the terminal_sync server
-TERMSYNC_TIMEOUT=10
+TERMSYNC_TIMEOUT=4
 
 
 # =============================================================================