Add Client.create_repository() [RHELDST-22483]

TODO

Author: rbikar

pubtools/pulplib/_impl/client/client.py

@@ -1054,3 +1054,34 @@ def _do_sync(self, repo_id, sync_options):
         return self._task_executor.submit(
             self._do_request, method="POST", url=url, json=body
         )
+
+    def create_repository(self, repo):
+        """
+        TODO ddocstring
+        """
+        url = os.path.join(self._url, "pulp/api/v2/repositories/")
+
+        body = repo._to_data()
+        repo_id = body["id"]
+
+        importer = body.pop("importers", [])
+        body["importer_type_id"] = importer[0]["importer_type_id"] if importer else None
+        body["importer_config"] = importer[0]["config"] if importer else None
+
+        def log_existing_repo(exception):
+            if (
+                getattr(exception, "response", None) is not None
+                and exception.response.status_code == 409
+            ):
+                LOG.warning("Repository %s already exists", repo_id)
+                return None
+
+            raise exception
+
+        LOG.debug("Creating repository %s", repo_id)
+        out = self._request_executor.submit(
+            self._do_request, method="POST", url=url, json=body
+        )
+
+        out = f_map(out, error_fn=log_existing_repo)
+        return f_map(out, lambda _: None)

pubtools/pulplib/_impl/client/retry.py

@@ -29,8 +29,8 @@ def should_retry(self, attempt, future):
 
         exception = future.exception()
         if exception and getattr(exception, "response", None) is not None:
-            # if returned status code is 404, never retry on that
-            if exception.response.status_code == 404:
+            # if returned status code is 404 or 409, never retry on that
+            if exception.response.status_code in (404, 409):
                 return False
 
         if exception and retry:

pubtools/pulplib/_impl/fake/client.py

@@ -651,3 +651,12 @@ def _do_sync(self, repo_id, sync_config):  # pylint:disable = unused-argument
             self._state.sync_history.append(Sync(repo_f.result(), [task], sync_config))
 
         return f_return([task])
+
+    def create_repository(self, repo):
+        with self._state.lock:
+            if repo.id not in [
+                existing_repo.id for existing_repo in self._state.repositories
+            ]:
+                self._state.repositories.append(repo)
+
+        return f_return(None)

pubtools/pulplib/_impl/model/common.py

@@ -234,3 +234,15 @@ def _delete(self, resource_type, resource_id):
         delete_f = client._delete_resource(resource_type, resource_id)
         delete_f = f_map(delete_f, self.__detach)
         return f_proxy(delete_f)
+
+
+def schemaless_init(cls, data):
+    # Construct and return an instance of (attrs-using) cls from
+    # pulp data, where data in pulp has no schema at all (and hence
+    # every field could possibly be missing).
+    kwargs = {}
+    for key in [fld.name for fld in attr.fields(cls)]:
+        if key in data:
+            kwargs[key] = data[key]
+
+    return cls(**kwargs)

pubtools/pulplib/_impl/model/repository/base.py

@@ -19,6 +19,7 @@
 from ...hooks import pm
 from ...util import dict_put, lookup, ABSENT
 
+from ..common import schemaless_init
 
 LOG = logging.getLogger("pubtools.pulplib")
 
@@ -35,6 +36,29 @@ def decorate(klass):
     return decorate
 
 
+@attr.s(kw_only=True, frozen=True)
+class Importer(object):
+    type_id = pulp_attrib(default=None, type=str, pulp_field="importer_type_id")
+
+    config = pulp_attrib(default=None, type=dict, pulp_field="importer_config")
+
+    @classmethod
+    def _from_data(cls, data):
+        # Convert from raw list/dict as provided in Pulp responses into model.
+        if isinstance(data, list):
+            return [cls._from_data(item) for item in data]
+
+        return schemaless_init(cls, data)
+
+    def _to_data(self):
+        return [
+            {
+                "importer_type_id": self.type_id,
+                "config": self.config,
+            }
+        ]
+
+
 @attr.s(kw_only=True, frozen=True)
 class PublishOptions(object):
     """Options controlling a repository
@@ -338,6 +362,19 @@ class Repository(PulpObject, Deletable):
 
     .. versionadded:: 2.37.0
     """
+    """
+    
+    """
+    importer = pulp_attrib(
+        default=Importer(),
+        type=Importer,
+        pulp_field="importers",
+        pulp_py_converter=Importer._from_data,
+        py_pulp_converter=Importer._to_data,
+    )
+    """
+    todo
+    """
 
     @distributors.validator
     def _check_repo_id(self, _, value):

pubtools/pulplib/_impl/model/repository/file.py

@@ -4,7 +4,7 @@
 from attr import validators
 from frozenlist2 import frozenlist
 
-from .base import Repository, SyncOptions, repo_type
+from .base import Repository, SyncOptions, repo_type, Importer
 from ...model.unit import FileUnit
 from ..attr import pulp_attrib
 from ... import compat_attr as attr
@@ -13,6 +13,13 @@
 LOG = logging.getLogger("pubtools.pulplib")
 
 
+@attr.s(kw_only=True, frozen=True)
+class IsoImporter(Importer):
+    type_id = pulp_attrib(
+        default="iso_importer", type=str, pulp_field="importer_type_id"
+    )
+
+
 @attr.s(kw_only=True, frozen=True)
 class FileSyncOptions(SyncOptions):
     """Options controlling a file repository
@@ -49,6 +56,14 @@ class FileRepository(Repository):
         converter=frozenlist,
     )
 
+    importer = pulp_attrib(
+        default=IsoImporter(),
+        type=IsoImporter,
+        pulp_field="importers",
+        pulp_py_converter=IsoImporter._from_data,
+        py_pulp_converter=IsoImporter._to_data,
+    )
+
     def upload_file(self, file_obj, relative_url=None, **kwargs):
         """Upload a file to this repository.
 

pubtools/pulplib/_impl/model/repository/yum.py

@@ -3,14 +3,21 @@
 from frozenlist2 import frozenlist
 
 from more_executors.futures import f_map, f_proxy, f_return, f_zip, f_flat_map
-from .base import Repository, SyncOptions, repo_type
+from .base import Repository, SyncOptions, repo_type, Importer
 from ..attr import pulp_attrib
 from ..common import DetachedException
 from ...model.unit import RpmUnit
 from ... import compat_attr as attr, comps
 from ...criteria import Criteria, Matcher
 
 
+@attr.s(kw_only=True, frozen=True)
+class YumImporter(Importer):
+    type_id = pulp_attrib(
+        default="yum_importer", type=str, pulp_field="importer_type_id"
+    )
+
+
 @attr.s(kw_only=True, frozen=True)
 class YumSyncOptions(SyncOptions):
     """Options controlling a yum repository
@@ -103,6 +110,14 @@ class YumRepository(Repository):
     )
     """Version of UBI config that should be used for population of this repository."""
 
+    importer = pulp_attrib(
+        default=YumImporter(),
+        type=YumImporter,
+        pulp_field="importers",
+        pulp_py_converter=YumImporter._from_data,
+        py_pulp_converter=YumImporter._to_data,
+    )
+
     def get_binary_repository(self):
         """Find and return the binary repository relating to this repository.
 

pubtools/pulplib/_impl/model/unit/base.py

@@ -139,15 +139,3 @@ def _usermeta_from_kwargs(cls, **kwargs):
             )
 
         return out
-
-
-def schemaless_init(cls, data):
-    # Construct and return an instance of (attrs-using) cls from
-    # pulp data, where data in pulp has no schema at all (and hence
-    # every field could possibly be missing).
-    kwargs = {}
-    for key in [fld.name for fld in attr.fields(cls)]:
-        if key in data:
-            kwargs[key] = data[key]
-
-    return cls(**kwargs)

pubtools/pulplib/_impl/model/unit/erratum.py

@@ -1,8 +1,9 @@
 from frozenlist2 import frozenlist
 
-from .base import Unit, PulpObject, unit_type, schemaless_init
+from .base import Unit, PulpObject, unit_type
 
 from ..attr import pulp_attrib
+from ..common import schemaless_init
 from ... import compat_attr as attr
 from ..validate import (
     optional_bool,

pubtools/pulplib/_impl/model/unit/modulemd.py

@@ -1,8 +1,9 @@
 import re
 
-from .base import Unit, unit_type, schemaless_init
+from .base import Unit, unit_type
 
 from ..attr import pulp_attrib
+from ..common import schemaless_init
 from ... import compat_attr as attr
 from ..convert import (
     frozenlist_or_none_converter,

pubtools/pulplib/_impl/model/unit/rpm.py

@@ -1,9 +1,10 @@
 import datetime
 
 from pubtools.pulplib._impl.model.validate import optional_list_of
-from .base import Unit, unit_type, schemaless_init
+from .base import Unit, unit_type
 
 from ..attr import pulp_attrib
+from ..common import schemaless_init
 from ... import compat_attr as attr
 from ..convert import (
     frozenlist_or_none_converter,

pubtools/pulplib/_impl/schema/repository.yaml

@@ -69,7 +69,9 @@ properties:
       # Note that this field is not set by Pulp itself.  Only certain tools
       # are expected to initialize this field when creating a repo.
       created:
-        type: string
+        anyOf:
+        - type: "null"
+        - type: string
         # example: 2019-06-05T11:56:50Z
         pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z"
 
@@ -100,7 +102,9 @@ properties:
 
       # Version of ubi config that should be used for population of this repository
       ubi_config_version:
-        type: string
+        anyOf:
+        - type: "null"
+        - type: string
 
       # Flag indicating whether the repository is visible in production instance
       # of download service. Stored as string.

tests/fake/test_fake_create_repo.py

@@ -0,0 +1,16 @@
+from pubtools.pulplib import FakeController, Repository
+
+
+def test_create_repository():
+    """Client.create_repository() with fake client adds new repositories to controller."""
+    controller = FakeController()
+
+    client = controller.client
+    client.create_repository(Repository(id="repo1"))
+    client.create_repository(Repository(id="repo2"))
+
+    # adding already existing repository has no effect
+    client.create_repository(Repository(id="repo1"))
+    # The change should be reflected in the controller,
+    # with two repositories present
+    assert controller.repositories == [Repository(id="repo1"), Repository(id="repo2")]

tests/repository/test_yum_repository.py

@@ -1,4 +1,5 @@
 import pytest
+import logging
 
 from pubtools.pulplib import Repository, YumRepository, DetachedException
 
@@ -193,3 +194,58 @@ def test_related_repositories_detached_client():
 
     with pytest.raises(DetachedException):
         repo_binary_test.get_binary_repository()
+
+
+def test_create_repository(client, requests_mocker):
+    repo = YumRepository(id="yum_repo")
+    repo.__dict__["_client"] = client
+
+    requests_mocker.post(
+        "https://pulp.example.com/pulp/api/v2/repositories/",
+        json={},
+    )
+
+    out = client.create_repository(repo)
+    # check return value of create_repository() call
+    assert out.result() is None
+
+    hist = requests_mocker.request_history
+    # there should be exactly one request sent
+    assert len(hist) == 1
+
+    query = hist[0].json()
+    # check id of repository sent in request body
+    assert query["id"] == "yum_repo"
+    # check importer data sent in request body that
+    # are automatically added for yum repository
+    assert query["importer_type_id"] == "yum_importer"
+    assert query["importer_config"] is None
+
+
+def test_create_repository_already_exists(client, requests_mocker, caplog):
+    repo = YumRepository(id="yum_repo")
+    repo.__dict__["_client"] = client
+
+    requests_mocker.post(
+        "https://pulp.example.com/pulp/api/v2/repositories/",
+        status_code=409,
+        text="Conflict 409 status",
+    )
+    with caplog.at_level(logging.WARNING):
+        out = client.create_repository(repo)
+        # check return value of create_repository() call
+        assert out.result() is None
+
+        hist = requests_mocker.request_history
+        # there should are exactly one request sent, 409 status is never retried
+        assert len(hist) == 1
+
+        query = hist[0].json()
+        # check id of repository sent in request body
+        assert query["id"] == "yum_repo"
+        # check importer data sent in request body that
+        # are automatically added for yum repository
+        assert query["importer_type_id"] == "yum_importer"
+        assert query["importer_config"] is None
+        # check logged information about existing repository
+        assert "Repository yum_repo already exists" in caplog.text