Add Client.create_repository() [RHELDST-22483]

`Client` is now capable of creating a repository on pulp
server. The repository is initialized with proper `Importer` in order
to enable sync/upload of content to new repositories.

Another notable changes:
* 409 status is not retried in requests as it is used as indicator of
 existing repository.
* Added create_repository() method to FakeClient as well.
* Small changes in the repository.yaml schema as some of the fields
  don't have to be necessarily initialized when creating repo but can be
  updated later.
* Added `Importer` pulp object mapping class and related subclasses
  that encapsulate minimal required fields for this object with valid
  defaults.
* added `"importers": True` to search options for repos
* fixed converter of `notes.signatures` on `Repository` - empty string is now
  converted to empty list, not to list with empty string

Author: rbikar

CHANGELOG.md

@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-- n/a
+- Added `Client.create_repository()` method
 
 ## [2.38.1] - 2023-12-14
 

docs/api/files.rst

@@ -14,6 +14,8 @@ Repository
 .. autoclass:: pubtools.pulplib.FileSyncOptions()
    :members:
 
+.. autoclass:: pubtools.pulplib.FileImporter()
+   :members:
 
 Units
 -----

docs/api/pulpcore.rst

@@ -27,6 +27,8 @@ Repository
 .. autoclass:: pubtools.pulplib.SyncOptions()
    :members:
 
+.. autoclass:: pubtools.pulplib.Importer()
+   :members:
 
 Units
 -----

docs/api/yum.rst

@@ -14,6 +14,8 @@ Repository
 .. autoclass:: pubtools.pulplib.YumSyncOptions()
    :members:
 
+.. autoclass:: pubtools.pulplib.YumImporter()
+   :members:
 
 Units: RPM
 ----------

pubtools/pulplib/__init__.py

@@ -31,5 +31,8 @@
     Task,
     MaintenanceReport,
     MaintenanceEntry,
+    Importer,
+    YumImporter,
+    FileImporter,
 )
 from ._impl.fake import FakeController

pubtools/pulplib/_impl/client/client.py

@@ -278,7 +278,7 @@ def search_repository(self, criteria=None):
                 Each page will contain a collection of
                 :class:`~pubtools.pulplib.Repository` objects.
         """
-        search_options = {"distributors": True}
+        search_options = {"distributors": True, "importers": True}
         return self._search(
             Repository, "repositories", criteria=criteria, search_options=search_options
         )
@@ -1054,3 +1054,80 @@ def _do_sync(self, repo_id, sync_options):
         return self._task_executor.submit(
             self._do_request, method="POST", url=url, json=body
         )
+
+    def create_repository(self, repo):
+        """Create a repository with initial data provided in the
+           argument. Importer for repository is automatically associated
+           if available. If repository already exists, warning is logged.
+           After repository has been successfully created or if repository already exists,
+           it is checked if expected and current repository configuration values are
+           correct.
+
+        Args:
+            repo (:class:`~pubtools.pulplib.Repository`)
+                A repository object used for creation.
+
+        Returns:
+            Future[Repository]
+                A future which is resolved with a value of ``Repository`` once the
+                repository has been created.
+
+        .. versionadded:: 2.39.0
+        """
+        url = os.path.join(self._url, "pulp/api/v2/repositories/")
+
+        body = repo._to_data()
+        repo_id = body["id"]
+
+        importer = body.pop("importers", [])
+        body["importer_type_id"] = importer[0]["importer_type_id"] if importer else None
+        body["importer_config"] = importer[0]["config"] if importer else None
+
+        def log_existing_repo(exception):
+            if (
+                getattr(exception, "response", None) is not None
+                and exception.response.status_code == 409
+            ):
+                LOG.warning("Repository %s already exists", repo_id)
+                return None
+
+            raise exception
+
+        def get_and_check_repo(_):
+            repo_on_server = self.get_repository(repo_id)
+            try:
+                assert (
+                    repo_on_server.result() == repo
+                ), "Repo exists on server with unexpected values"
+            except AssertionError:
+                if importer:
+                    for attr in ["type_id", "config"]:
+                        expected = getattr(repo.importer, attr)
+                        current = getattr(repo_on_server.importer, attr)
+                        if expected != current:
+                            LOG.error(
+                                "Repository %s contains wrong importer %s\n"
+                                "\t expected: %s\n"
+                                "\t current: %s\n",
+                                repo_id,
+                                attr,
+                                expected,
+                                current,
+                            )
+                LOG.exception(
+                    "Repository %s exists on server and contains unexpected values",
+                    repo_id,
+                )
+                raise
+
+            return repo_on_server
+
+        LOG.debug("Creating repository %s", repo_id)
+        out = self._request_executor.submit(
+            self._do_request, method="POST", url=url, json=body
+        )
+
+        out = f_map(out, error_fn=log_existing_repo)
+        out = f_flat_map(out, get_and_check_repo)
+
+        return f_proxy(out)

pubtools/pulplib/_impl/client/retry.py

@@ -29,8 +29,8 @@ def should_retry(self, attempt, future):
 
         exception = future.exception()
         if exception and getattr(exception, "response", None) is not None:
-            # if returned status code is 404, never retry on that
-            if exception.response.status_code == 404:
+            # if returned status code is 404 or 409, never retry on that
+            if exception.response.status_code in (404, 409):
                 return False
 
         if exception and retry:

pubtools/pulplib/_impl/fake/client.py

@@ -651,3 +651,12 @@ def _do_sync(self, repo_id, sync_config):  # pylint:disable = unused-argument
             self._state.sync_history.append(Sync(repo_f.result(), [task], sync_config))
 
         return f_return([task])
+
+    def create_repository(self, repo):
+        with self._state.lock:
+            if repo.id not in [
+                existing_repo.id for existing_repo in self._state.repositories
+            ]:
+                self._state.repositories.append(repo)
+
+        return self.get_repository(repo.id)

pubtools/pulplib/_impl/model/__init__.py

@@ -9,6 +9,9 @@
     FileSyncOptions,
     ContainerSyncOptions,
     YumSyncOptions,
+    Importer,
+    FileImporter,
+    YumImporter,
 )
 from .unit import (
     Unit,

pubtools/pulplib/_impl/model/common.py

@@ -234,3 +234,15 @@ def _delete(self, resource_type, resource_id):
         delete_f = client._delete_resource(resource_type, resource_id)
         delete_f = f_map(delete_f, self.__detach)
         return f_proxy(delete_f)
+
+
+def schemaless_init(cls, data):
+    # Construct and return an instance of (attrs-using) cls from
+    # pulp data, where data in pulp has no schema at all (and hence
+    # every field could possibly be missing).
+    kwargs = {}
+    for key in [fld.name for fld in attr.fields(cls)]:
+        if key in data:
+            kwargs[key] = data[key]
+
+    return cls(**kwargs)

pubtools/pulplib/_impl/model/repository/__init__.py

@@ -1,4 +1,4 @@
-from .base import Repository, PublishOptions, SyncOptions
+from .base import Repository, PublishOptions, SyncOptions, Importer
 from .container import ContainerImageRepository, ContainerSyncOptions
-from .yum import YumRepository, YumSyncOptions
-from .file import FileRepository, FileSyncOptions
+from .yum import YumRepository, YumSyncOptions, YumImporter
+from .file import FileRepository, FileSyncOptions, FileImporter

pubtools/pulplib/_impl/model/repository/base.py

@@ -8,17 +8,19 @@
 from frozenlist2 import frozenlist
 from more_executors.futures import f_proxy, f_map, f_flat_map
 
+from frozendict.core import frozendict  # pylint: disable=no-name-in-module
 from .repo_lock import RepoLock
 from ..attr import pulp_attrib, PULP2_FIELD, PULP2_MUTABLE
 from ..common import PulpObject, Deletable, DetachedException
-from ..convert import frozenlist_or_none_converter
+from ..convert import frozenlist_or_none_converter, frozendict_or_none_converter
 from ..distributor import Distributor
 from ...criteria import Criteria, Matcher
 from ...schema import load_schema
 from ... import compat_attr as attr
 from ...hooks import pm
 from ...util import dict_put, lookup, ABSENT
 
+from ..common import schemaless_init
 
 LOG = logging.getLogger("pubtools.pulplib")
 
@@ -35,6 +37,44 @@ def decorate(klass):
     return decorate
 
 
+@attr.s(kw_only=True, frozen=True)
+class Importer(PulpObject):
+    """
+    Importer is a pulp object that needs to be associated with repository
+    in order to successfully sync or upload content to it.
+    """
+
+    type_id = pulp_attrib(default=None, type=str, pulp_field="importer_type_id")
+    """
+    Type id of the importer.
+    """
+    config = pulp_attrib(
+        default=attr.Factory(frozendict),
+        type=frozendict,
+        converter=frozendict_or_none_converter,
+        pulp_field="config",
+    )
+    """
+    Configuration dictionary of the importer.
+    """
+
+    @classmethod
+    def _from_data(cls, data):
+        # Convert from raw list/dict as provided in Pulp responses into model.
+        if isinstance(data, list):
+            return cls._from_data(data[0]) if data else schemaless_init(cls, data)
+
+        return schemaless_init(cls, data)
+
+    def _to_data(self):
+        return [
+            {
+                "importer_type_id": self.type_id,
+                "config": self.config,
+            }
+        ]
+
+
 @attr.s(kw_only=True, frozen=True)
 class PublishOptions(object):
     """Options controlling a repository
@@ -249,7 +289,7 @@ class Repository(PulpObject, Deletable):
         default=attr.Factory(frozenlist),
         type=list,
         pulp_field="notes.signatures",
-        pulp_py_converter=lambda sigs: sigs.split(","),
+        pulp_py_converter=lambda sigs: sigs.split(",") if sigs else [],
         py_pulp_converter=",".join,
         converter=lambda keys: frozenlist([k.strip() for k in keys]),
     )
@@ -339,6 +379,19 @@ class Repository(PulpObject, Deletable):
     .. versionadded:: 2.37.0
     """
 
+    importer = pulp_attrib(
+        default=Importer(),
+        type=Importer,
+        pulp_field="importers",
+        pulp_py_converter=Importer._from_data,
+        py_pulp_converter=Importer._to_data,
+    )
+    """
+    An object of :class:`~pubtools.pulplib.Importer` that is associated with the repository.
+
+    .. versionadded:: 2.39.0
+    """
+
     @distributors.validator
     def _check_repo_id(self, _, value):
         # checks if distributor's repository id is same as the repository it

pubtools/pulplib/_impl/model/repository/file.py

@@ -4,7 +4,7 @@
 from attr import validators
 from frozenlist2 import frozenlist
 
-from .base import Repository, SyncOptions, repo_type
+from .base import Repository, SyncOptions, repo_type, Importer
 from ...model.unit import FileUnit
 from ..attr import pulp_attrib
 from ... import compat_attr as attr
@@ -13,6 +13,16 @@
 LOG = logging.getLogger("pubtools.pulplib")
 
 
+@attr.s(kw_only=True, frozen=True)
+class FileImporter(Importer):
+    type_id = pulp_attrib(
+        default="iso_importer", type=str, pulp_field="importer_type_id"
+    )
+    """
+    Specific importer_type_id for File repositories.
+    """
+
+
 @attr.s(kw_only=True, frozen=True)
 class FileSyncOptions(SyncOptions):
     """Options controlling a file repository
@@ -49,6 +59,19 @@ class FileRepository(Repository):
         converter=frozenlist,
     )
 
+    importer = pulp_attrib(
+        default=FileImporter(),
+        type=FileImporter,
+        pulp_field="importers",
+        pulp_py_converter=FileImporter._from_data,
+        py_pulp_converter=FileImporter._to_data,
+    )
+    """
+    An object of :class:`~pubtools.pulplib.FileImporter` that is associated with the repository.
+
+    .. versionadded:: 2.39.0 
+    """
+
     def upload_file(self, file_obj, relative_url=None, **kwargs):
         """Upload a file to this repository.
 

pubtools/pulplib/_impl/model/repository/yum.py

@@ -3,14 +3,24 @@
 from frozenlist2 import frozenlist
 
 from more_executors.futures import f_map, f_proxy, f_return, f_zip, f_flat_map
-from .base import Repository, SyncOptions, repo_type
+from .base import Repository, SyncOptions, repo_type, Importer
 from ..attr import pulp_attrib
 from ..common import DetachedException
 from ...model.unit import RpmUnit
 from ... import compat_attr as attr, comps
 from ...criteria import Criteria, Matcher
 
 
+@attr.s(kw_only=True, frozen=True)
+class YumImporter(Importer):
+    type_id = pulp_attrib(
+        default="yum_importer", type=str, pulp_field="importer_type_id"
+    )
+    """
+    Specific importer_type_id for Yum repositories.
+    """
+
+
 @attr.s(kw_only=True, frozen=True)
 class YumSyncOptions(SyncOptions):
     """Options controlling a yum repository
@@ -103,6 +113,19 @@ class YumRepository(Repository):
     )
     """Version of UBI config that should be used for population of this repository."""
 
+    importer = pulp_attrib(
+        default=YumImporter(),
+        type=YumImporter,
+        pulp_field="importers",
+        pulp_py_converter=YumImporter._from_data,
+        py_pulp_converter=YumImporter._to_data,
+    )
+    """
+    An object of :class:`~pubtools.pulplib.YumImporter` that is associated with the repository.
+
+    .. versionadded:: 2.39.0
+    """
+
     def get_binary_repository(self):
         """Find and return the binary repository relating to this repository.
 

pubtools/pulplib/_impl/model/unit/base.py

@@ -139,15 +139,3 @@ def _usermeta_from_kwargs(cls, **kwargs):
             )
 
         return out
-
-
-def schemaless_init(cls, data):
-    # Construct and return an instance of (attrs-using) cls from
-    # pulp data, where data in pulp has no schema at all (and hence
-    # every field could possibly be missing).
-    kwargs = {}
-    for key in [fld.name for fld in attr.fields(cls)]:
-        if key in data:
-            kwargs[key] = data[key]
-
-    return cls(**kwargs)