feat(schematic): fds 1447: implement pagination (#2421)

.github/workflows/schematic-api-ci.yml

@@ -125,4 +125,4 @@ jobs:
             && nx affected --target=test-all"
 
       - name: Remove the dev container
-        run: docker rm -f sage_devcontainer
+        run: docker rm -f sage_devcontainer

apps/schematic/api/.openapi-generator/FILES

@@ -12,32 +12,35 @@ schematic_api/models/__init__.py
 schematic_api/models/asset_type.py
 schematic_api/models/base_model_.py
 schematic_api/models/basic_error.py
-schematic_api/models/connected_nodes.py
-schematic_api/models/connected_nodes_page.py
-schematic_api/models/connected_nodes_page_all_of.py
-schematic_api/models/dataset.py
-schematic_api/models/datasets_page.py
-schematic_api/models/datasets_page_all_of.py
-schematic_api/models/file.py
-schematic_api/models/files_page.py
-schematic_api/models/files_page_all_of.py
-schematic_api/models/manifest.py
+schematic_api/models/connected_node_pair.py
+schematic_api/models/connected_node_pair_array.py
+schematic_api/models/connected_node_pair_page.py
+schematic_api/models/connected_node_pair_page_all_of.py
+schematic_api/models/dataset_metadata.py
+schematic_api/models/dataset_metadata_array.py
+schematic_api/models/dataset_metadata_page.py
+schematic_api/models/dataset_metadata_page_all_of.py
+schematic_api/models/file_metadata.py
+schematic_api/models/file_metadata_array.py
+schematic_api/models/file_metadata_page.py
+schematic_api/models/file_metadata_page_all_of.py
+schematic_api/models/manifest_metadata.py
+schematic_api/models/manifest_metadata_array.py
+schematic_api/models/manifest_metadata_page.py
+schematic_api/models/manifest_metadata_page_all_of.py
 schematic_api/models/manifest_validation_result.py
-schematic_api/models/manifests_page.py
-schematic_api/models/manifests_page_all_of.py
 schematic_api/models/node.py
-schematic_api/models/node_properties_page.py
-schematic_api/models/node_properties_page_all_of.py
-schematic_api/models/node_property.py
-schematic_api/models/nodes_page.py
-schematic_api/models/nodes_page_all_of.py
+schematic_api/models/node_array.py
+schematic_api/models/node_page.py
+schematic_api/models/node_page_all_of.py
+schematic_api/models/node_property_array.py
 schematic_api/models/page_metadata.py
-schematic_api/models/project.py
-schematic_api/models/projects_page.py
-schematic_api/models/projects_page_all_of.py
+schematic_api/models/project_metadata.py
+schematic_api/models/project_metadata_array.py
+schematic_api/models/project_metadata_page.py
+schematic_api/models/project_metadata_page_all_of.py
 schematic_api/models/validation_rule.py
-schematic_api/models/validation_rules_page.py
-schematic_api/models/validation_rules_page_all_of.py
+schematic_api/models/validation_rule_array.py
 schematic_api/openapi/openapi.yaml
 schematic_api/test/__init__.py
 schematic_api/typing_utils.py

apps/schematic/api/schematic_api/controllers/paging.py

@@ -0,0 +1,97 @@
+"""Functionality to handle paginated endpoints"""
+
+import math
+from typing import TypeVar
+
+ITEM_TYPE = TypeVar("ITEM_TYPE")
+TOTAL_ITEMS_MSG = "total_items must be 0 or greater: "
+PAGE_MAX_ITEMS_MSG = "page_max_items must be 1 or greater: "
+PAGE_NUMBER_MSG = "page_number must be 1 or greater: "
+
+
+class Page:
+    """This represents a page for a generic list of items for a paginated endpoint"""
+
+    def __init__(
+        self, items: list[ITEM_TYPE], page_number: int = 1, page_max_items: int = 100000
+    ) -> None:
+        """
+        Args:
+            items (list[ITEM_TYPE]): A list of all items in the query
+            page_number (int, optional): The page number the current request is for. Defaults to 1.
+            page_max_items (int, optional): The maximum number of items per page. Defaults to 100000.
+        """
+        self.page_number = page_number
+        self.page_max_items = page_max_items
+        self.total_items = len(items)
+        self.total_pages = get_page_amount(self.total_items, page_max_items)
+        self.has_next = page_number < self.total_pages
+        self.has_previous = page_number > 1
+        self.items: list[ITEM_TYPE] = get_item_slice(items, page_max_items, page_number)
+
+
+def get_page_amount(total_items: int, page_max_items: int) -> int:
+    """Getes the amount of pages total based on the number of items and page size
+
+    Args:
+        total_items (int): The total number of items in the query
+        page_max_items (int): The maximum number of items per page
+
+    Raises:
+        ValueError: total_items is less than 0
+        ValueError: page_max_items is less than
+
+    Returns:
+        int: The amount of pages
+    """
+    if total_items < 0:
+        raise ValueError(TOTAL_ITEMS_MSG, total_items)
+    if page_max_items < 1:
+        raise ValueError(PAGE_MAX_ITEMS_MSG, page_max_items)
+    return math.ceil(total_items / page_max_items)
+
+
+def get_item_slice(
+    items: list[ITEM_TYPE], page_max_items: int, page_number: int
+) -> list[ITEM_TYPE]:
+    """Gets a list slice based on the paging parameters
+
+    Args:
+        items (list[ITEM_TYPE]): A list of items to be sliced
+        page_max_items (int): The maximum number of items per page
+        page_number (int): The page number the current request is for
+
+    Returns:
+        list[ITEM_TYPE]: The slice of items
+    """
+    page_indeces = get_page_indeces(len(items), page_max_items, page_number)
+    return items[page_indeces[0] : page_indeces[1]]
+
+
+def get_page_indeces(
+    total_items: int, page_max_items: int, page_number: int
+) -> tuple[int, int]:
+    """Gets the indces used to slice the list of items
+
+    Args:
+        total_items (int): The total number of items in the query
+        page_max_items (int): The maximum number of items per page
+        page_number (int): The page number the current request is for
+
+    Raises:
+        ValueError: total_items is less than 0
+        ValueError: page_max_items is less than 1
+        ValueError: page_number is less than 1
+
+    Returns:
+        tuple[int, int]: The two indeces to slice the list of items with
+    """
+    if total_items < 0:
+        raise ValueError(TOTAL_ITEMS_MSG, total_items)
+    if page_max_items < 1:
+        raise ValueError(PAGE_MAX_ITEMS_MSG, page_max_items)
+    if page_number < 1:
+        raise ValueError(PAGE_NUMBER_MSG, page_number)
+    index1 = (page_number - 1) * page_max_items
+    index2 = min(index1 + page_max_items, total_items)
+    return (index1, index2)

apps/schematic/api/schematic_api/controllers/schema_controller.py

@@ -5,10 +5,16 @@
 from typing import Union
 
 from schematic_api.models.basic_error import BasicError  # noqa: E501
-from schematic_api.models.connected_nodes_page import ConnectedNodesPage  # noqa: E501
-from schematic_api.models.node_properties_page import NodePropertiesPage  # noqa: E501
-from schematic_api.models.nodes_page import NodesPage  # noqa: E501
-from schematic_api.models.validation_rules_page import ValidationRulesPage  # noqa: E501
+from schematic_api.models.connected_node_pair_array import (
+    ConnectedNodePairArray,
+)  # noqa: E501
+from schematic_api.models.connected_node_pair_page import (
+    ConnectedNodePairPage,
+)  # noqa: E501
+from schematic_api.models.node_array import NodeArray  # noqa: E501
+from schematic_api.models.node_page import NodePage  # noqa: E501
+from schematic_api.models.node_property_array import NodePropertyArray  # noqa: E501
+from schematic_api.models.validation_rule_array import ValidationRuleArray  # noqa: E501
 from schematic_api import util
 from schematic_api.controllers import schema_controller_impl
 
@@ -32,19 +38,104 @@ def get_component(component_label, schema_url, include_index=None):  # noqa: E50
     )
 
 
-def get_connected_nodes(schema_url, relationship_type):  # noqa: E501
-    """Gets a list of connected node pairs
+def get_connected_node_pair_array(schema_url, relationship_type):  # noqa: E501
+    """Gets an array of connected node pairs
 
-    Gets a list of connected node pairs # noqa: E501
+    Gets a array of connected node pairs # noqa: E501
 
     :param schema_url: The URL of a schema in jsonld form
     :type schema_url: str
     :param relationship_type: Type of relationship in a schema, such as requiresDependency
     :type relationship_type: str
 
-    :rtype: Union[ConnectedNodesPage, Tuple[ConnectedNodesPage, int], Tuple[ConnectedNodesPage, int, Dict[str, str]]
+    :rtype: Union[ConnectedNodePairArray, Tuple[ConnectedNodePairArray, int], Tuple[ConnectedNodePairArray, int, Dict[str, str]]
     """
-    return schema_controller_impl.get_connected_nodes(schema_url, relationship_type)
+    return schema_controller_impl.get_connected_node_pair_array(
+        schema_url, relationship_type
+    )
+
+
+def get_connected_node_pair_page(
+    schema_url, relationship_type, page_number=None, page_max_items=None
+):  # noqa: E501
+    """Gets a page of connected node pairs
+
+    Gets a page of connected node pairs # noqa: E501
+
+    :param schema_url: The URL of a schema in jsonld form
+    :type schema_url: str
+    :param relationship_type: Type of relationship in a schema, such as requiresDependency
+    :type relationship_type: str
+    :param page_number: The page number to get for a paginated query
+    :type page_number: int
+    :param page_max_items: The maximum number of items per page (up to 100,000) for paginated endpoints
+    :type page_max_items: int
+
+    :rtype: Union[ConnectedNodePairPage, Tuple[ConnectedNodePairPage, int], Tuple[ConnectedNodePairPage, int, Dict[str, str]]
+    """
+    return schema_controller_impl.get_connected_node_pair_page(
+        schema_url, relationship_type, page_number, page_max_items
+    )
+
+
+def get_node_dependency_array(
+    node_label, schema_url, return_display_names=None, return_ordered_by_schema=None
+):  # noqa: E501
+    """Gets the immediate dependencies that are related to the given source node
+
+    Gets the immediate dependencies that are related to the given source node # noqa: E501
+
+    :param node_label: The label of the source node in a schema to get the dependencies of
+    :type node_label: str
+    :param schema_url: The URL of a schema in jsonld form
+    :type schema_url: str
+    :param return_display_names: Whether or not to return the display names of the component, otherwise the label
+    :type return_display_names: bool
+    :param return_ordered_by_schema: Whether or not to order the components by their order in the schema, otherwise random
+    :type return_ordered_by_schema: bool
+
+    :rtype: Union[NodeArray, Tuple[NodeArray, int], Tuple[NodeArray, int, Dict[str, str]]
+    """
+    return schema_controller_impl.get_node_dependency_array(
+        node_label, schema_url, return_display_names, return_ordered_by_schema
+    )
+
+
+def get_node_dependency_page(
+    node_label,
+    schema_url,
+    return_display_names=None,
+    return_ordered_by_schema=None,
+    page_number=None,
+    page_max_items=None,
+):  # noqa: E501
+    """Gets the immediate dependencies that are related to the given source node
+
+    Gets the immediate dependencies that are related to the given source node # noqa: E501
+
+    :param node_label: The label of the source node in a schema to get the dependencies of
+    :type node_label: str
+    :param schema_url: The URL of a schema in jsonld form
+    :type schema_url: str
+    :param return_display_names: Whether or not to return the display names of the component, otherwise the label
+    :type return_display_names: bool
+    :param return_ordered_by_schema: Whether or not to order the components by their order in the schema, otherwise random
+    :type return_ordered_by_schema: bool
+    :param page_number: The page number to get for a paginated query
+    :type page_number: int
+    :param page_max_items: The maximum number of items per page (up to 100,000) for paginated endpoints
+    :type page_max_items: int
+
+    :rtype: Union[NodePage, Tuple[NodePage, int], Tuple[NodePage, int, Dict[str, str]]
+    """
+    return schema_controller_impl.get_node_dependency_page(
+        node_label,
+        schema_url,
+        return_display_names,
+        return_ordered_by_schema,
+        page_number,
+        page_max_items,
+    )
 
 
 def get_node_is_required(node_display, schema_url):  # noqa: E501
@@ -72,11 +163,26 @@ def get_node_properties(node_label, schema_url):  # noqa: E501
     :param schema_url: The URL of a schema in jsonld form
     :type schema_url: str
 
-    :rtype: Union[NodePropertiesPage, Tuple[NodePropertiesPage, int], Tuple[NodePropertiesPage, int, Dict[str, str]]
+    :rtype: Union[NodePropertyArray, Tuple[NodePropertyArray, int], Tuple[NodePropertyArray, int, Dict[str, str]]
     """
     return schema_controller_impl.get_node_properties(node_label, schema_url)
 
 
+def get_node_validation_rules(node_display, schema_url):  # noqa: E501
+    """Gets the validation rules, along with the arguments for each given rule associated with a given node
+
+    Gets the validation rules, along with the arguments for each given rule associated with a given node # noqa: E501
+
+    :param node_display: The display name of the node in a schema
+    :type node_display: str
+    :param schema_url: The URL of a schema in jsonld form
+    :type schema_url: str
+
+    :rtype: Union[ValidationRuleArray, Tuple[ValidationRuleArray, int], Tuple[ValidationRuleArray, int, Dict[str, str]]
+    """
+    return schema_controller_impl.get_node_validation_rules(node_display, schema_url)
+
+
 def get_property_label(
     node_display, schema_url, use_strict_camel_case=None
 ):  # noqa: E501
@@ -109,41 +215,3 @@ def get_schema_attributes(schema_url):  # noqa: E501
     :rtype: Union[str, Tuple[str, int], Tuple[str, int, Dict[str, str]]
     """
     return schema_controller_impl.get_schema_attributes(schema_url)
-
-
-def list_node_dependencies(
-    node_label, schema_url, return_display_names=None, return_ordered_by_schema=None
-):  # noqa: E501
-    """Gets the immediate dependencies that are related to the given source node
-
-    Gets the immediate dependencies that are related to the given source node # noqa: E501
-
-    :param node_label: The label of the source node in a schema to get the dependencies of
-    :type node_label: str
-    :param schema_url: The URL of a schema in jsonld form
-    :type schema_url: str
-    :param return_display_names: Whether or not to return the display names of the component, otherwise the label
-    :type return_display_names: bool
-    :param return_ordered_by_schema: Whether or not to order the components by their order in the schema, otherwise random
-    :type return_ordered_by_schema: bool
-
-    :rtype: Union[NodesPage, Tuple[NodesPage, int], Tuple[NodesPage, int, Dict[str, str]]
-    """
-    return schema_controller_impl.list_node_dependencies(
-        node_label, schema_url, return_display_names, return_ordered_by_schema
-    )
-
-
-def list_node_validation_rules(node_display, schema_url):  # noqa: E501
-    """Gets the validation rules, along with the arguments for each given rule associated with a given node
-
-    Gets the validation rules, along with the arguments for each given rule associated with a given node # noqa: E501
-
-    :param node_display: The display name of the node in a schema
-    :type node_display: str
-    :param schema_url: The URL of a schema in jsonld form
-    :type schema_url: str
-
-    :rtype: Union[ValidationRulesPage, Tuple[ValidationRulesPage, int], Tuple[ValidationRulesPage, int, Dict[str, str]]
-    """
-    return schema_controller_impl.list_node_validation_rules(node_display, schema_url)