add endpoint

.github/workflows/schematic-api-ci.yml

@@ -120,6 +120,8 @@ jobs:
         shell: python
 
       - name: Test the affected projects (all)
+        env:
+          SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
         run: |
           devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
             && nx affected --target=test-all"

apps/schematic/api/.gitignore

@@ -70,6 +70,7 @@ target/
 #secrets
 *secrets*
 synapse_config.yaml
+schematic_api/schematic_service_account_creds.json
 
 #schematic downloaded files
 manifests

apps/schematic/api/.openapi-generator-ignore

@@ -32,9 +32,10 @@ requirements.txt
 schematic_api/__main__.py
 setup.py
 test-requirements.txt
+schematic_api/test/test_manifest_generation.py
 schematic_api/test/test_storage_controller.py
 schematic_api/test/test_schema_controller.py
 schematic_api/test/test_manifest_validation_controller.py
 schematic_api/test/test_versions_controller.py
 schematic_api/test/test_tangled_tree_controller.py
-tox.ini
+tox.ini

apps/schematic/api/.openapi-generator/FILES

@@ -1,6 +1,7 @@
 AUTHORS.md
 schematic_api/__init__.py
 schematic_api/controllers/__init__.py
+schematic_api/controllers/manifest_generation_controller.py
 schematic_api/controllers/manifest_validation_controller.py
 schematic_api/controllers/schema_controller.py
 schematic_api/controllers/security_controller_.py
@@ -24,6 +25,7 @@ schematic_api/models/file_metadata.py
 schematic_api/models/file_metadata_array.py
 schematic_api/models/file_metadata_page.py
 schematic_api/models/file_metadata_page_all_of.py
+schematic_api/models/google_sheet_links.py
 schematic_api/models/manifest_metadata.py
 schematic_api/models/manifest_metadata_array.py
 schematic_api/models/manifest_metadata_page.py
@@ -43,5 +45,6 @@ schematic_api/models/validation_rule.py
 schematic_api/models/validation_rule_array.py
 schematic_api/openapi/openapi.yaml
 schematic_api/test/__init__.py
+schematic_api/test/test_manifest_generation_controller.py
 schematic_api/typing_utils.py
 schematic_api/util.py

apps/schematic/api/schematic_api/config.yml

@@ -1,36 +1,6 @@
-# Do not change the 'definitions' section unless you know what you're doing
-definitions:
-  synapse_config: ".synapseConfig"
-  service_acct_creds: "schematic_service_account_creds.json"
+asset_store:
+  synapse:
+    config: '.synapseConfig'
 
-synapse:
-  master_fileview: 'syn23643253'
-  manifest_folder: 'manifests'
-  manifest_basename: 'synapse_storage_manifest' 
-  service_acct_creds: 'syn25171627'
-
-manifest:
-  # if making many manifests, just include name prefix
-  title: 'example'
-  # to make all manifests enter only 'all manifests'
-  data_type: 
-    - 'Biospecimen'
-    - 'Patient'
-
-model:
-  input:
-    location: 'tests/data/example.model.jsonld'
-    file_type: 'local'
-
-style:
-  google_manifest:
-    req_bg_color:
-      red: 0.9215
-      green: 0.9725
-      blue: 0.9803
-    opt_bg_color:
-      red: 1.0
-      green: 1.0
-      blue: 0.9019
-    master_template_id: '1LYS5qE4nV9jzcYw5sXwCza25slDfRA1CIg3cs-hCdpU'
-    strict_validation: true
+google_sheets:
+  service_acct_creds: 'schematic_service_account_creds.json'

apps/schematic/api/schematic_api/controllers/manifest_generation_controller.py

@@ -0,0 +1,55 @@
+import connexion
+import six
+from typing import Dict
+from typing import Tuple
+from typing import Union
+
+from schematic_api.models.basic_error import BasicError  # noqa: E501
+from schematic_api.models.google_sheet_links import GoogleSheetLinks  # noqa: E501
+from schematic_api import util
+from schematic_api.controllers import manifest_generation_controller_impl
+
+
+def generate_google_sheet_manifests(
+    schema_url,
+    asset_view_id,
+    add_annotations=None,
+    dataset_id_array=None,
+    manifest_title=None,
+    node_label_array=None,
+    use_strict_validation=None,
+    generate_all_manifests=None,
+):  # noqa: E501
+    """Generates a list of google sheet links
+
+    Generates a list of google sheet links # noqa: E501
+
+    :param schema_url: The URL of a schema in jsonld form
+    :type schema_url: str
+    :param asset_view_id: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project
+    :type asset_view_id: str
+    :param add_annotations: If true, annotations are added to the manifest
+    :type add_annotations: bool
+    :param dataset_id_array: An array of dataset ids
+    :type dataset_id_array: List[str]
+    :param manifest_title: If making one manifest, the title of the manifest If making multiple manifests, the prefix of the title of the manifests
+    :type manifest_title: str
+    :param node_label_array: An array of nodel labels
+    :type node_label_array: List[str]
+    :param use_strict_validation: If true, users are blocked from entering incorrect values. If false, users will get a warning when using incorrect values.
+    :type use_strict_validation: bool
+    :param generate_all_manifests: If true, a manifest for all components will be generated, datasetIds will be ignored If false, manifests for each id in datasetIds will be generated
+    :type generate_all_manifests: bool
+
+    :rtype: Union[GoogleSheetLinks, Tuple[GoogleSheetLinks, int], Tuple[GoogleSheetLinks, int, Dict[str, str]]
+    """
+    return manifest_generation_controller_impl.generate_google_sheet_manifests(
+        schema_url,
+        asset_view_id,
+        add_annotations,
+        dataset_id_array,
+        manifest_title,
+        node_label_array,
+        use_strict_validation,
+        generate_all_manifests,
+    )

apps/schematic/api/schematic_api/controllers/manifest_generation_controller_impl.py

@@ -0,0 +1,104 @@
+"""Manifest generation functions"""
+# pylint: disable=too-many-arguments
+
+from schematic import CONFIG  # type: ignore
+from schematic.manifest.generator import ManifestGenerator  # type: ignore
+
+from schematic_api.models.basic_error import BasicError
+from schematic_api.models.google_sheet_links import GoogleSheetLinks
+from schematic_api.controllers.utils import (
+    handle_exceptions,
+    get_access_token,
+    download_schema_file_as_jsonld,
+    InvalidValueError,
+)
+
+
+@handle_exceptions
+def generate_google_sheet_manifests(
+    schema_url: str,
+    asset_view_id: str,
+    add_annotations: bool,
+    dataset_id_array: list[str] | None,
+    manifest_title: str,
+    node_label_array: list[str] | None,
+    use_strict_validation: bool,
+    generate_all_manifests: bool,
+) -> tuple[GoogleSheetLinks | BasicError, int]:
+    """Generates a list of links to manifets in google sheet form
+
+    Args:
+        schema_url (str): The URL of the schema
+        dataset_id_array (list[str] | None): Use this to get the existing manifests in the
+          datasets. Must be of same type as the node_label_array, same order, and same length
+        asset_view_id (str): ID of the asset view
+        node_label_array (list[str] | None): The datatypes of the manifests to generate
+        add_annotations (bool): Whether or not annotatiosn get added to the manifest
+        manifest_title (str): Title of the manifest
+        use_strict_validation (bool): Whether or not to use google sheet strict validation
+        generate_all_manifests (bool): Will generate a manifest for all data types
+
+    Raises:
+        ValueError: When generate_all_manifests is true and either dataset_id_array or
+          node_label_array are provided
+        ValueError: When generate_all_manifests is false and node_label_array is not provided
+        ValueError: When generate_all_manifests is false and dataset_id_arrayy is provided,
+          but it doesn't match the length of node_label_array
+
+    Returns:
+        tuple[GoogleSheetLinks | BasicError, int]: A tuple
+           The first item is either the google sheet links of the manifests or an error object
+           The second item is the response status
+    """
+
+    if generate_all_manifests:
+        if dataset_id_array:
+            raise InvalidValueError(
+                "When generate_all_manifests is True dataset_id_array must be None",
+                {"dataset_id_array": dataset_id_array},
+            )
+        if node_label_array:
+            raise InvalidValueError(
+                "When generate_all_manifests is True node_label_array must be None",
+                {"node_label_array": node_label_array},
+            )
+        node_label_array = ["all_manifests"]
+
+    else:
+        if not node_label_array:
+            raise InvalidValueError(
+                (
+                    "When generate_all_manifests is False node_label_array must be a list with "
+                    "atleast one item"
+                ),
+                {"node_label_array": node_label_array},
+            )
+        if dataset_id_array and len(dataset_id_array) != len(node_label_array):
+            raise InvalidValueError(
+                (
+                    "When generate_all_manifests is False node_label_array and dataset_id_array "
+                    "must both lists with the same length"
+                ),
+                {
+                    "node_label_array": node_label_array,
+                    "dataset_id_array": dataset_id_array,
+                },
+            )
+
+    access_token = get_access_token()
+    CONFIG.load_config("schematic_api/config.yml")
+    CONFIG.synapse_master_fileview_id = asset_view_id
+    schema_path = download_schema_file_as_jsonld(schema_url)
+    links = ManifestGenerator.create_manifests(
+        jsonld=schema_path,
+        output_format="google_sheet",
+        data_types=node_label_array,
+        title=manifest_title,
+        access_token=access_token,
+        dataset_ids=dataset_id_array,
+        strict=use_strict_validation,
+        use_annotations=add_annotations,
+    )
+    result: GoogleSheetLinks | BasicError = GoogleSheetLinks(links)
+    status = 200
+    return result, status

apps/schematic/api/schematic_api/controllers/utils.py

@@ -65,6 +65,11 @@ def func(*args: Any, **kwargs: Any) -> tuple[Any | BasicError, int]:
             res = BasicError("Invalid URL", status, str(error))
             return res, status
 
+        except InvalidValueError as error:
+            status = 422
+            res = BasicError("Invalid data", status, str(error))
+            return res, status
+
         except Exception as error:  # pylint: disable=broad-exception-caught
             status = 500
             res = BasicError("Internal error", status, str(error))
@@ -90,6 +95,25 @@ def __str__(self) -> str:
         return f"{self.message}: {self.url}"
 
 
+class InvalidValueError(Exception):
+    """Raised when a provided value for an endpoint is invalid"""
+
+    def __init__(self, message: str, values: dict[str, Any]):
+        """
+        Args:
+            message (str): The error message
+            values (dict[str, Any]): A dict where the argument names are keys and
+              argument values are values
+        """
+        self.message = message
+        self.values = values
+
+        super().__init__(self.message)
+
+    def __str__(self) -> str:
+        return f"{self.message}: {self.values}"
+
+
 def download_schema_file_as_jsonld(schema_url: str) -> str:
     """Downloads a schema and saves it as temp file
 

apps/schematic/api/schematic_api/models/__init__.py

@@ -17,6 +17,7 @@
 from schematic_api.models.file_metadata_array import FileMetadataArray
 from schematic_api.models.file_metadata_page import FileMetadataPage
 from schematic_api.models.file_metadata_page_all_of import FileMetadataPageAllOf
+from schematic_api.models.google_sheet_links import GoogleSheetLinks
 from schematic_api.models.manifest_metadata import ManifestMetadata
 from schematic_api.models.manifest_metadata_array import ManifestMetadataArray
 from schematic_api.models.manifest_metadata_page import ManifestMetadataPage

apps/schematic/api/schematic_api/models/google_sheet_links.py

@@ -0,0 +1,64 @@
+# coding: utf-8
+
+from __future__ import absolute_import
+from datetime import date, datetime  # noqa: F401
+
+from typing import List, Dict  # noqa: F401
+
+from schematic_api.models.base_model_ import Model
+from schematic_api import util
+
+
+class GoogleSheetLinks(Model):
+    """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+
+    Do not edit the class manually.
+    """
+
+    def __init__(self, links=None):  # noqa: E501
+        """GoogleSheetLinks - a model defined in OpenAPI
+
+        :param links: The links of this GoogleSheetLinks.  # noqa: E501
+        :type links: List[str]
+        """
+        self.openapi_types = {
+            'links': List[str]
+        }
+
+        self.attribute_map = {
+            'links': 'links'
+        }
+
+        self._links = links
+
+    @classmethod
+    def from_dict(cls, dikt) -> 'GoogleSheetLinks':
+        """Returns the dict as a model
+
+        :param dikt: A dict.
+        :type: dict
+        :return: The GoogleSheetLinks of this GoogleSheetLinks.  # noqa: E501
+        :rtype: GoogleSheetLinks
+        """
+        return util.deserialize_model(dikt, cls)
+
+    @property
+    def links(self):
+        """Gets the links of this GoogleSheetLinks.
+
+
+        :return: The links of this GoogleSheetLinks.
+        :rtype: List[str]
+        """
+        return self._links
+
+    @links.setter
+    def links(self, links):
+        """Sets the links of this GoogleSheetLinks.
+
+
+        :param links: The links of this GoogleSheetLinks.
+        :type links: List[str]
+        """
+
+        self._links = links