Add content reader to decouple NFS file reads [RHELDST-26339]

Reading content of the push items strongly depends on their presence
as file on NFS or locally-mounted filesystem. However, with advent
of new push item sources, it likely that the push items might not be
stored as a file or be locally available.

Hence, this PR adds an `opener` to the pushitems that should fetch
the corresponding bit stream which will be accessible from `content()`.
`opener` should be defined in the `Source` while creating the pushitems
to get the bits from the specific content source. PushItem is not bound
to have an `opener` and `content()` will return the bits as file object
only when the `opener` is defined.

This adds a provision to transfer the responsibility of fetching the
content from the consumer that is using the content/pushitems to the
`Source` and corresponding `pushitem` here, abstracting the reading
mechanism from the user and provide flexibility to fetch from different
locations/protocols.

Author: rajulkumar

src/pushsource/_impl/list_cmd.py

@@ -36,6 +36,10 @@
 
 LOG = logging.getLogger("pushsource-ls")
 
+EXCLUDED_ATTRIBUTES = [
+    "opener",
+]
+
 
 class ItemDumper(yaml.SafeDumper):
     # Custom dumper adding support for any types appearing on pushitems
@@ -77,7 +81,13 @@ def format_python_black(item):
 
 
 def format_yaml(item):
-    data = {type(item).__name__: attr.asdict(item, recurse=True)}
+    data = {
+        type(item).__name__: attr.asdict(
+            item,
+            recurse=True,
+            filter=lambda attribute, _: attribute.name not in EXCLUDED_ATTRIBUTES,
+        )
+    }
     return yaml.dump([data], Dumper=ItemDumper)
 
 

src/pushsource/_impl/model/base.py

@@ -16,6 +16,7 @@
     optional,
     optional_str,
 )
+from ..reader import PushItemReader
 
 
 LOG = logging.getLogger("pushsource")
@@ -178,6 +179,14 @@ def _default_build_info(self):
     doesn't enforce this.
     """
 
+    opener = attr.ib(type=callable, default=None, repr=False)
+    """The opener, when given a push item, should return a file-like object
+    suitable for reading this item's bytes. The object can be retrieved from
+    :meth:`~pushsource.PushItem.content()` method.
+
+    .. versionadded:: 2.51.0
+    """
+
     def with_checksums(self):
         """Return a copy of this push item with checksums present.
 
@@ -248,3 +257,36 @@ def with_checksums(self):
             updated_sums[attribute] = hasher.hexdigest()
 
         return attr.evolve(self, **updated_sums)
+
+    def content(self):
+        """Returns a read-only, non-seekable content of this push item.
+
+        For push items representing a single file, content will obtain a stream
+        for reading that file's bytes.
+        For example, ``RpmPushItem.content`` can be used to read the content of
+        an RPM; ``VMIPushItem.content`` can be used to read the content of a VMI
+        and so on.
+
+        Not every type of push item can be read in this way.
+        For example, a single ``ContainerImagePushItem`` may represent any number
+        of artifacts making up a container image, to be accessed from a container
+        image registry in the usual way - not using this method. Other types of
+        push items such as ``ErratumPushItem`` may be understood as metadata-only
+        items and do not themselves have any content. For items such as these,
+        this method will return None.
+
+
+        Returns:
+            :class:`~io.BufferedReader`
+                A non-seekable object of the push item content
+            ``None``
+                If the :attr:`~pushsource.PushItem.opener` in the pushitem is not defined to read
+                the content.
+
+        .. versionadded:: 2.51.0
+        """
+        return (
+            PushItemReader(self.opener(self), self.src or self.name)
+            if self.opener
+            else None
+        )

src/pushsource/_impl/model/comps.py

@@ -1,5 +1,6 @@
 from .base import PushItem
 from .. import compat_attr as attr
+from ..utils.openers import open_src_local
 
 
 @attr.s()
@@ -12,3 +13,11 @@ class CompsXmlPushItem(PushItem):
 
     This library does not verify that the referenced file is valid.
     """
+
+    opener = attr.ib(type=callable, default=open_src_local, repr=False)
+    """Identical to :attr:`~pushsource.PushItem.opener`.
+
+    This defaults to reading content as file from :attr:`~pushsource.PushItem.src`
+    
+    .. versionadded:: 2.51.0
+    """

src/pushsource/_impl/model/directory.py

@@ -1,5 +1,6 @@
 from .base import PushItem
 from .. import compat_attr as attr
+from ..utils.openers import open_src_local
 
 
 @attr.s()
@@ -9,3 +10,11 @@ class DirectoryPushItem(PushItem):
     On a directory push item, the src attribute contains the full path to a directory tree.
     It should generally be interpreted as a request to recursively publish that entire directory
     tree as-is."""
+
+    opener = attr.ib(type=callable, default=open_src_local, repr=False)
+    """Identical to :attr:`~pushsource.PushItem.opener`.
+
+    This defaults to reading content as file from :attr:`~pushsource.PushItem.src`
+    
+    .. versionadded:: 2.51.0
+    """

src/pushsource/_impl/model/file.py

@@ -1,6 +1,7 @@
 from .base import PushItem
 from .. import compat_attr as attr
 from .conv import optional_str, convert_maybe
+from ..utils.openers import open_src_local
 
 
 @attr.s()
@@ -52,3 +53,11 @@ def _check_order(self, _, value):
         # - This check will also filter out NaN.
         if not -99999 <= value <= 99999:
             raise ValueError("display_order must be within range -99999 .. 99999")
+
+    opener = attr.ib(type=callable, default=open_src_local, repr=False)
+    """Identical to :attr:`~pushsource.PushItem.opener`.
+
+    This defaults to reading content as file from :attr:`~pushsource.PushItem.src`
+    
+    .. versionadded:: 2.51.0
+    """

src/pushsource/_impl/model/modulemd.py

@@ -1,5 +1,6 @@
 from .base import PushItem
 from .. import compat_attr as attr
+from ..utils.openers import open_src_local
 
 
 @attr.s()
@@ -14,6 +15,14 @@ class ModuleMdPushItem(PushItem):
     modulemd stream.
     """
 
+    opener = attr.ib(type=callable, default=open_src_local, repr=False)
+    """Identical to :attr:`~pushsource.PushItem.opener`.
+
+    This defaults to reading content as file from :attr:`~pushsource.PushItem.src`
+    
+    .. versionadded:: 2.51.0
+    """
+
 
 @attr.s()
 class ModuleMdSourcePushItem(PushItem):
@@ -25,3 +34,11 @@ class ModuleMdSourcePushItem(PushItem):
     This library does not verify that the referenced file is a valid
     modulemd source document.
     """
+
+    opener = attr.ib(type=callable, default=open_src_local, repr=False)
+    """Identical to :attr:`~pushsource.PushItem.opener`.
+
+    This defaults to reading content as file from :attr:`~pushsource.PushItem.src`
+    
+    .. versionadded:: 2.51.0
+    """

src/pushsource/_impl/model/productid.py

@@ -7,6 +7,7 @@
 from .base import PushItem
 from .conv import convert_maybe, sloppylist
 from .. import compat_attr as attr
+from ..utils.openers import open_src_local
 
 
 # Red Hat OID namespace is "1.3.6.1.4.1.2312.9",
@@ -114,3 +115,11 @@ def _load_products(self, path):
             )
             result.append(product)
         return result
+
+    opener = attr.ib(type=callable, default=open_src_local, repr=False)
+    """Identical to :attr:`~pushsource.PushItem.opener`.
+
+    This defaults to reading content as file from :attr:`~pushsource.PushItem.src`
+    
+    .. versionadded:: 2.51.0
+    """

src/pushsource/_impl/model/rpm.py

@@ -1,6 +1,7 @@
 from .base import PushItem
 from .conv import optional_str
 from .. import compat_attr as attr
+from ..utils.openers import open_src_local
 
 
 @attr.s()
@@ -32,3 +33,11 @@ class RpmPushItem(PushItem):
     .. _ghc-8.4-820200708061905.9edba152: https://koji.fedoraproject.org/koji/buildinfo?buildID=1767200
     .. _ghc-8.4.4-74.module_el8+12161+cf1bd7f2: https://koji.fedoraproject.org/koji/buildinfo?buildID=1767130
     """
+
+    opener = attr.ib(type=callable, default=open_src_local, repr=False)
+    """Identical to :attr:`~pushsource.PushItem.opener`.
+
+    This defaults to reading content as file from :attr:`~pushsource.PushItem.src`
+    
+    .. versionadded:: 2.51.0
+    """

src/pushsource/_impl/reader.py

@@ -0,0 +1,19 @@
+from io import BufferedReader, SEEK_SET, UnsupportedOperation
+
+
+class PushItemReader(BufferedReader):
+    # Internal class to ensure that the file-like content object returned by
+    # the push items are read-only and non-seekable with a name attribute.
+    def __init__(self, raw, name, **kwargs):
+        super(PushItemReader, self).__init__(raw, **kwargs)
+        self._name = name
+
+    @property
+    def name(self):
+        return self._name
+
+    def seekable(self):
+        return False
+
+    def seek(self, offset, whence=SEEK_SET):
+        raise UnsupportedOperation(f"Seek unsupported while reading {self.name}")

src/pushsource/_impl/utils/openers.py

@@ -0,0 +1,5 @@
+def open_src_local(item):
+    # default opener for the push items
+    # assumes that the item's 'src' points to the
+    # locally-accessible file
+    return open(item.src, "rb")

tests/baseline/cases/direct-cgw.yml

@@ -15,6 +15,7 @@ items:
     - dest
     md5sum: null
     name: cgw.yaml
+    opener: null
     origin: direct
     sha256sum: null
     signing_key: null

tests/baseline/cases/direct-comps.yml

@@ -14,6 +14,7 @@ items:
     dest: []
     md5sum: null
     name: mycomps.xml
+    opener: open_src_local
     origin: direct
     sha256sum: null
     signing_key: null

tests/baseline/cases/direct-dir.yml

@@ -15,6 +15,7 @@ items:
     - /destdir
     md5sum: null
     name: srcdir
+    opener: open_src_local
     origin: direct
     sha256sum: null
     signing_key: null

tests/baseline/cases/direct-file.yml

@@ -18,6 +18,7 @@ items:
     display_order: null
     md5sum: null
     name: custom-filename
+    opener: open_src_local
     origin: direct
     sha256sum: null
     signing_key: null

tests/baseline/cases/direct-modulemd-src.yml

@@ -14,6 +14,7 @@ items:
     dest: []
     md5sum: null
     name: modules.src.txt
+    opener: open_src_local
     origin: direct
     sha256sum: null
     signing_key: null

tests/baseline/cases/direct-modulemd.yml

@@ -14,6 +14,7 @@ items:
     dest: []
     md5sum: null
     name: my-best-module
+    opener: open_src_local
     origin: direct
     sha256sum: null
     signing_key: null

tests/baseline/cases/direct-productid.yml

@@ -17,6 +17,7 @@ items:
     - repo3
     md5sum: null
     name: some-cert
+    opener: open_src_local
     origin: direct
     products:
     - architecture:

tests/baseline/cases/direct-rpm.yml

@@ -15,6 +15,7 @@ items:
     md5sum: null
     module_build: null
     name: test.rpm
+    opener: open_src_local
     origin: custom-origin
     sha256sum: null
     signing_key: A1B2C3