Remove flask_uploads.py (#152)

* WIP: remove flask_uploads

* Cleanup

* Add option for remote sequence storage

* Formatting

* Remove legacy comment

* Update workflow versions

* Fix permission check issues

* Add more permission checks

Author: thomas-bc

.github/workflows/fprime-gds-tests.yml

@@ -16,9 +16,9 @@ jobs:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies

src/fprime_gds/common/files/uplinker.py

@@ -47,7 +47,9 @@ def __init__(self, uplinker):
         self.queue = queue.Queue()
         self.__file_store = []
         self.__exit = threading.Event()
-        self.__thread = threading.Thread(target=self.run, name="UplinkerThread", args=())
+        self.__thread = threading.Thread(
+            target=self.run, name="UplinkerThread", args=()
+        )
         self.__thread.start()
 
     def enqueue(self, filepath, destination):
@@ -228,9 +230,7 @@ def start(self, file_obj):
         # Prevent multiple uplinks at once
         if self.state != FileStates.IDLE:
             msg = f"Currently uplinking file '{self.active.source}' cannot start uplinking '{file_obj.source}'"
-            raise FileUplinkerBusyException(
-                msg
-            )
+            raise FileUplinkerBusyException(msg)
         self.state = FileStates.RUNNING
         self.active = file_obj
         self.active.open(TransmitFileState.READ)

src/fprime_gds/common/pipeline/files.py

@@ -7,7 +7,7 @@
 
 @author mstarch
 """
-import os
+from pathlib import Path
 import fprime_gds.common.files.downlinker
 import fprime_gds.common.files.uplinker
 
@@ -43,10 +43,11 @@ def setup_file_handling(
         )
         file_decoder.register(self.__downlinker)
         distributor.register("FW_PACKET_HAND", self.__uplinker)
-        if not os.access(down_store, os.W_OK):
+        try:
+            Path(down_store).mkdir(parents=True, exist_ok=True)
+        except PermissionError:
             raise PermissionError(
-                f"{down_store} is not writable. Downlinker not be able to save files. "
-                "Fix permissions or change storage directory with --file-storage-directory."
+                f"{down_store} is not writable. Fix permissions or change storage directory with --file-storage-directory."
             )
 
     @property

src/fprime_gds/common/pipeline/standard.py

@@ -44,6 +44,8 @@ def __init__(self):
         self.client_socket = None
         self.logger = None
         self.dictionary_path = None
+        self.up_store = None
+        self.down_store = None
 
         self.__dictionaries = dictionaries.Dictionaries()
         self.__coders = encoding.EncodingDecoding()
@@ -52,19 +54,31 @@ def __init__(self):
         self.__transport_type = ThreadedTCPSocketClient
 
     def setup(
-        self, config, dictionary, down_store, logging_prefix=None, packet_spec=None
+        self, config, dictionary, file_store, logging_prefix=None, packet_spec=None
     ):
         """
         Setup the standard pipeline for moving data from the middleware layer through the GDS layers using the standard
         patterns. This allows just registering the consumers, and invoking 'setup' all other of the GDS support layer.
 
         :param config: config object used when constructing the pipeline.
         :param dictionary: dictionary path. Used to setup loading of dictionaries.
-        :param down_store: downlink storage directory
+        :param file_store: uplink/downlink storage directory
         :param logging_prefix: logging prefix. Defaults to not logging at all.
         :param packet_spec: location of packetized telemetry XML specification.
         """
-        assert dictionary is not None and Path(dictionary).is_file(), f"Dictionary {dictionary} does not exist"
+        assert (
+            dictionary is not None and Path(dictionary).is_file()
+        ), f"Dictionary {dictionary} does not exist"
+        # File storage configuration for uplink and downlink
+        self.up_store = Path(file_store) / "fprime-uplink"
+        self.down_store = Path(file_store) / "fprime-downlink"
+        try:
+            self.down_store.mkdir(parents=True, exist_ok=True)
+            self.up_store.mkdir(parents=True, exist_ok=True)
+        except PermissionError:
+            raise PermissionError(
+                f"{file_store} is not writable. Fix permissions or change storage directory with --file-storage-directory."
+            )
         self.dictionary_path = Path(dictionary)
         # Loads the distributor and client socket
         self.distributor = fprime_gds.common.distributor.distributor.Distributor(config)
@@ -76,7 +90,7 @@ def setup(
         )
         self.histories.setup_histories(self.coders)
         self.files.setup_file_handling(
-            down_store,
+            self.down_store,
             self.coders.file_encoder,
             self.coders.file_decoder,
             self.distributor,
@@ -152,7 +166,11 @@ def connect(
             outgoing_tag: this pipeline will produce data for supplied tag (FSW, GUI). Default: FSW
         """
         # Backwards compatibility with the old method .connect(host, port)
-        if isinstance(incoming_tag, int) and ":" not in connection_uri and outgoing_tag == RoutingTag.FSW:
+        if (
+            isinstance(incoming_tag, int)
+            and ":" not in connection_uri
+            and outgoing_tag == RoutingTag.FSW
+        ):
             connection_uri = f"{connection_uri}:{incoming_tag}"
             incoming_tag = RoutingTag.GUI
         self.client_socket.connect(connection_uri, incoming_tag, outgoing_tag)

src/fprime_gds/executables/cli.py

@@ -547,18 +547,31 @@ def get_arguments(self) -> Dict[Tuple[str, ...], Dict[str, Any]]:
 
         return {
             ("--file-storage-directory",): {
-                "dest": "files_directory",
+                "dest": "files_storage_directory",
                 "action": "store",
-                "default": "/tmp/" + username + "/fprime-downlink/",
+                "default": "/tmp/" + username,
                 "required": False,
                 "type": str,
-                "help": "File to store uplink and downlink files. Default: %(default)s",
-            }
+                "help": "Directory to store uplink and downlink files. Default: %(default)s",
+            },
+            ("--remote-sequence-directory",): {
+                "dest": "remote_sequence_directory",
+                "action": "store",
+                "default": "/seq",
+                "required": False,
+                "type": str,
+                "help": "Directory to save command sequence binaries, on the remote FSW. Default: %(default)s",
+            },
         }
 
     def handle_arguments(self, args, **kwargs):
         """Handle arguments as parsed"""
-        os.makedirs(args.files_directory, exist_ok=True)
+        try:
+            Path(args.files_storage_directory).mkdir(parents=True, exist_ok=True)
+        except PermissionError:
+            raise PermissionError(
+                f"{args.files_storage_directory} is not writable. Fix permissions or change storage directory with --file-storage-directory."
+            )
         return args
 
 
@@ -584,7 +597,7 @@ def pipeline_factory(args_ns, pipeline=None) -> StandardPipeline:
         pipeline_arguments = {
             "config": ConfigManager(),
             "dictionary": args_ns.dictionary,
-            "down_store": args_ns.files_directory,
+            "file_store": args_ns.files_storage_directory,
             "packet_spec": args_ns.packet_spec,
             "logging_prefix": args_ns.logs,
         }

src/fprime_gds/flask/app.py

@@ -30,7 +30,6 @@
 import fprime_gds.flask.stats
 import fprime_gds.flask.updown
 from fprime_gds.executables.cli import ParserBase, StandardPipelineParser
-from fprime_gds.flask import flask_uploads
 
 from . import components
 
@@ -49,8 +48,7 @@ def construct_app():
     2. Setup JSON encoding for Flask and flask_restful to handle F prime types natively
     3. Setup standard pipeline used throughout the system
     4. Create Restful API for registering flask items
-    5. Setup flask_uploads settings
-    6. Register all restful endpoints
+    5. Register all restful endpoints
 
     :return: setup app
     """
@@ -77,9 +75,6 @@ def construct_app():
 
     # Restful API registration
     api = fprime_gds.flask.errors.setup_error_handling(app)
-    # File upload configuration, 1 set for everything
-    uplink_set = flask_uploads.UploadSet("uplink", flask_uploads.ALL)
-    flask_uploads.configure_uploads(app, [uplink_set])
 
     # Application routes
     api.add_resource(
@@ -137,7 +132,7 @@ def construct_app():
     api.add_resource(
         fprime_gds.flask.updown.FileUploads,
         "/upload/files",
-        resource_class_args=[pipeline.files.uplinker, uplink_set],
+        resource_class_args=[pipeline.files.uplinker, pipeline.up_store],
     )
     api.add_resource(
         fprime_gds.flask.updown.FileDownload,
@@ -150,9 +145,9 @@ def construct_app():
         "/sequence",
         resource_class_args=[
             args_ns.dictionary,
-            app.config["UPLOADED_UPLINK_DEST"],
+            pipeline.up_store,
             pipeline.files.uplinker,
-            app.config["REMOTE_SEQ_DIRECTORY"],
+            args_ns.remote_sequence_directory,
         ],
     )
     api.add_resource(

src/fprime_gds/flask/default_settings.py

@@ -9,23 +9,12 @@
 #
 ####
 import os
-import getpass
-
-# Select uploads directory and create it
-username = getpass.getuser()
-uplink_dir = os.environ.get("UP_FILES_DIR", "/tmp/" + username + "/fprime-uplink/")
-DOWNLINK_DIR = os.environ.get("DOWN_FILES_DIR", "/tmp/" + username + "/fprime-downlink/")
 
 STANDARD_PIPELINE_ARGUMENTS = os.environ.get("STANDARD_PIPELINE_ARGUMENTS").split("|")
 
 SERVE_LOGS = os.environ.get("SERVE_LOGS", "YES") == "YES"
-UPLOADED_UPLINK_DEST = uplink_dir
-UPLOADS_DEFAULT_DEST = uplink_dir
-REMOTE_SEQ_DIRECTORY = "/seq"
-MAX_CONTENT_LENGTH = 32 * 1024 * 1024  # Max length of request is 32MiB
 
+MAX_CONTENT_LENGTH = 32 * 1024 * 1024  # Max length of request is 32MiB
 
-for directory in [UPLOADED_UPLINK_DEST, UPLOADS_DEFAULT_DEST, DOWNLINK_DIR]:
-    os.makedirs(directory, exist_ok=True)
 
 # TODO: load real config