Merge pull request pulp#673 from gerrod3/domains

Add domain support

Author: gerrod3

.github/workflows/scripts/install.sh

@@ -97,7 +97,7 @@ if [ "$TEST" = "s3" ]; then
   sed -i -e '$a s3_test: true\
 minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"\
-pulp_scenario_settings: null\
+pulp_scenario_settings: {"domain_enabled": true}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
   export PULP_API_ROOT="/rerouted/djnd/"
@@ -111,7 +111,7 @@ if [ "$TEST" = "azure" ]; then
       - ./azurite:/etc/pulp\
     command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml
   sed -i -e '$a azure_test: true\
-pulp_scenario_settings: null\
+pulp_scenario_settings: {"domain_enabled": true}\
 pulp_scenario_env: {}\
 ' vars/main.yaml
 fi

CHANGES/668.feature

@@ -0,0 +1 @@
+Added Domain support.

docs/tech-preview.rst

@@ -9,3 +9,4 @@ The following features are currently being released as part of a tech preview
 * Fully mirror Python repositories provided PyPI and Pulp itself.
 * ``Twine`` upload packages to indexes at endpoints '/simple` or '/legacy'.
 * Create pull-through caches of remote sources.
+* Pulp Domain support

pulp_python/app/__init__.py

@@ -10,3 +10,4 @@ class PulpPythonPluginAppConfig(PulpPluginAppConfig):
     label = "python"
     version = "3.12.0.dev"
     python_package_name = "pulp-python"
+    domain_compatible = True

pulp_python/app/migrations/0012_add_domain.py

@@ -0,0 +1,37 @@
+# Generated by Django 4.2.10 on 2024-05-30 17:53
+
+from django.db import migrations, models
+import django.db.models.deletion
+import pulpcore.app.util
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("python", "0011_alter_pythondistribution_distribution_ptr_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="pythonpackagecontent",
+            unique_together=set(),
+        ),
+        migrations.AddField(
+            model_name="pythonpackagecontent",
+            name="_pulp_domain",
+            field=models.ForeignKey(
+                default=pulpcore.app.util.get_domain_pk,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="core.domain",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="pythonpackagecontent",
+            name="sha256",
+            field=models.CharField(db_index=True, max_length=64),
+        ),
+        migrations.AlterUniqueTogether(
+            name="pythonpackagecontent",
+            unique_together={("sha256", "_pulp_domain")},
+        ),
+    ]

pulp_python/app/modelresource.py

@@ -1,5 +1,6 @@
 from pulpcore.plugin.importexport import BaseContentResource
 from pulpcore.plugin.modelresources import RepositoryResource
+from pulpcore.plugin.util import get_domain
 from pulp_python.app.models import (
     PythonPackageContent,
     PythonRepository,
@@ -15,7 +16,9 @@ def set_up_queryset(self):
         """
         :return: PythonPackageContent specific to a specified repo-version.
         """
-        return PythonPackageContent.objects.filter(pk__in=self.repo_version.content)
+        return PythonPackageContent.objects.filter(
+            pk__in=self.repo_version.content, _pulp_domain=get_domain()
+        )
 
     class Meta:
         model = PythonPackageContent

pulp_python/app/models.py

@@ -23,6 +23,7 @@
     PYPI_SERIAL_CONSTANT,
 )
 from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version
+from pulpcore.plugin.util import get_domain_pk, get_domain
 
 log = getLogger(__name__)
 
@@ -68,6 +69,7 @@ def content_handler(self, path):
         path = PurePath(path)
         name = None
         version = None
+        domain = get_domain()
         if path.match("pypi/*/*/json"):
             version = path.parts[2]
             name = path.parts[1]
@@ -76,7 +78,7 @@ def content_handler(self, path):
         elif len(path.parts) and path.parts[0] == "simple":
             # Temporary fix for PublishedMetadata not being properly served from remote storage
             # https://github.com/pulp/pulp_python/issues/413
-            if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
+            if domain.storage_class != "pulpcore.app.models.storage.FileSystem":
                 if self.publication or self.repository:
                     try:
                         publication = self.publication or Publication.objects.filter(
@@ -105,7 +107,11 @@ def content_handler(self, path):
             )
             # TODO Change this value to the Repo's serial value when implemented
             headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)}
-            json_body = python_content_to_json(self.base_path, package_content, version=version)
+            if not settings.DOMAIN_ENABLED:
+                domain = None
+            json_body = python_content_to_json(
+                self.base_path, package_content, version=version, domain=domain
+            )
             if json_body:
                 return json_response(json_body, headers=headers)
 
@@ -143,7 +149,7 @@ class PythonPackageContent(Content):
     name = models.TextField()
     name.register_lookup(NormalizeName)
     version = models.TextField()
-    sha256 = models.CharField(unique=True, db_index=True, max_length=64)
+    sha256 = models.CharField(db_index=True, max_length=64)
     # Optional metadata
     python_version = models.TextField()
     metadata_version = models.TextField()
@@ -168,6 +174,8 @@ class PythonPackageContent(Content):
     classifiers = models.JSONField(default=list)
     project_urls = models.JSONField(default=dict)
     description_content_type = models.TextField()
+    # Pulp Domains
+    _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT)
 
     @staticmethod
     def init_from_artifact_and_relative_path(artifact, relative_path):
@@ -179,6 +187,8 @@ def init_from_artifact_and_relative_path(artifact, relative_path):
         data["version"] = metadata.version
         data["filename"] = path.name
         data["sha256"] = artifact.sha256
+        data["pulp_domain_id"] = artifact.pulp_domain_id
+        data["_pulp_domain_id"] = artifact.pulp_domain_id
         return PythonPackageContent(**data)
 
     def __str__(self):
@@ -200,7 +210,7 @@ def __str__(self):
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
-        unique_together = ("sha256",)
+        unique_together = ("sha256", "_pulp_domain")
 
 
 class PythonPublication(Publication):

pulp_python/app/pypi/serializers.py

@@ -4,6 +4,7 @@
 from rest_framework import serializers
 from pulp_python.app.utils import DIST_EXTENSIONS
 from pulpcore.plugin.models import Artifact
+from pulpcore.plugin.util import get_domain
 from django.db.utils import IntegrityError
 
 log = logging.getLogger(__name__)
@@ -76,7 +77,7 @@ def validate(self, data):
         try:
             artifact.save()
         except IntegrityError:
-            artifact = Artifact.objects.get(sha256=artifact.sha256)
+            artifact = Artifact.objects.get(sha256=artifact.sha256, pulp_domain=get_domain())
             artifact.touch()
             log.info(f"Artifact for {file.name} already existed in database")
         data["content"] = (artifact, file.name)

pulp_python/app/pypi/views.py

@@ -26,7 +26,9 @@
 from pathlib import PurePath
 from pypi_simple.parse_stream import parse_links_stream_response
 
+from pulpcore.plugin.viewsets import OperationPostponedResponse
 from pulpcore.plugin.tasking import dispatch
+from pulpcore.plugin.util import get_domain
 from pulp_python.app.models import (
     PythonDistribution,
     PythonPackageContent,
@@ -75,7 +77,7 @@ def get_distribution(path):
             "repository", "publication", "publication__repository_version", "remote"
         )
         try:
-            return distro_qs.get(base_path=path)
+            return distro_qs.get(base_path=path, pulp_domain=get_domain())
         except ObjectDoesNotExist:
             raise Http404(f"No PythonDistribution found for base_path {path}")
 
@@ -100,6 +102,14 @@ def get_drvc(self, path):
         content = self.get_content(repo_ver)
         return distro, repo_ver, content
 
+    def initial(self, request, *args, **kwargs):
+        """Perform common initialization tasks for PyPI endpoints."""
+        super().initial(request, *args, **kwargs)
+        if settings.DOMAIN_ENABLED:
+            self.base_content_url = urljoin(BASE_CONTENT_URL, f"{get_domain().name}/")
+        else:
+            self.base_content_url = BASE_CONTENT_URL
+
 
 class PackageUploadMixin(PyPIMixin):
     """A Mixin to provide package upload support."""
@@ -137,7 +147,7 @@ def upload(self, request, path):
                           kwargs={"artifact_sha256": artifact.sha256,
                                   "filename": filename,
                                   "repository_pk": str(repo.pk)})
-        return Response(data={"task": reverse('tasks-detail', args=[result.pk], request=None)})
+        return OperationPostponedResponse(result, request)
 
     def upload_package_group(self, repo, artifact, filename, session):
         """Steps 4 & 5, spawns tasks to add packages to index."""
@@ -176,7 +186,7 @@ def create_group_upload_task(self, cur_session, repository, artifact, filename,
         return reverse('tasks-detail', args=[result.pk], request=None)
 
 
-class SimpleView(ViewSet, PackageUploadMixin):
+class SimpleView(PackageUploadMixin, ViewSet):
     """View for the PyPI simple API."""
 
     permission_classes = [IsAuthenticatedOrReadOnly]
@@ -186,7 +196,7 @@ def list(self, request, path):
         """Gets the simple api html page for the index."""
         distro, repo_version, content = self.get_drvc(path)
         if self.should_redirect(distro, repo_version=repo_version):
-            return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/'))
+            return redirect(urljoin(self.base_content_url, f'{path}/simple/'))
         names = content.order_by('name').values_list('name', flat=True).distinct().iterator()
         return StreamingHttpResponse(write_simple_index(names, streamed=True))
 
@@ -197,7 +207,7 @@ def parse_url(link):
             digest, _, value = parsed.fragment.partition('=')
             stripped_url = urlunsplit(chain(parsed[:3], ("", "")))
             redirect = f'{path}/{link.text}?redirect={stripped_url}'
-            d_url = urljoin(BASE_CONTENT_URL, redirect)
+            d_url = urljoin(self.base_content_url, redirect)
             return link.text, d_url, value if digest == 'sha256' else ''
 
         url = remote.get_remote_artifact_url(f'simple/{package}/')
@@ -224,7 +234,7 @@ def retrieve(self, request, path, package):
             if not repo_ver or not content.filter(name__normalize=normalized).exists():
                 return self.pull_through_package_simple(normalized, path, distro.remote)
         if self.should_redirect(distro, repo_version=repo_ver):
-            return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/{normalized}/'))
+            return redirect(urljoin(self.base_content_url, f'{path}/simple/{normalized}/'))
         packages = (
             content.filter(name__normalize=normalized)
             .values_list('filename', 'sha256', 'name')
@@ -237,7 +247,7 @@ def retrieve(self, request, path, package):
         else:
             packages = chain([present], packages)
             name = present[2]
-        releases = ((f, urljoin(BASE_CONTENT_URL, f'{path}/{f}'), d) for f, d, _ in packages)
+        releases = ((f, urljoin(self.base_content_url, f'{path}/{f}'), d) for f, d, _ in packages)
         return StreamingHttpResponse(write_simple_detail(name, releases, streamed=True))
 
     @extend_schema(request=PackageUploadSerializer,
@@ -253,7 +263,7 @@ def create(self, request, path):
         return self.upload(request, path)
 
 
-class MetadataView(ViewSet, PyPIMixin):
+class MetadataView(PyPIMixin, ViewSet):
     """View for the PyPI JSON metadata endpoint."""
 
     authentication_classes = []
@@ -272,6 +282,7 @@ def retrieve(self, request, path, meta):
         meta_path = PurePath(meta)
         name = None
         version = None
+        domain = None
         if meta_path.match("*/*/json"):
             version = meta_path.parts[1]
             name = meta_path.parts[0]
@@ -281,13 +292,17 @@ def retrieve(self, request, path, meta):
             package_content = content.filter(name__iexact=name)
             # TODO Change this value to the Repo's serial value when implemented
             headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)}
-            json_body = python_content_to_json(path, package_content, version=version)
+            if settings.DOMAIN_ENABLED:
+                domain = get_domain()
+            json_body = python_content_to_json(
+                path, package_content, version=version, domain=domain
+            )
             if json_body:
                 return Response(data=json_body, headers=headers)
         return Response(status="404")
 
 
-class PyPIView(ViewSet, PyPIMixin):
+class PyPIView(PyPIMixin, ViewSet):
     """View for base_url of distribution."""
 
     authentication_classes = []
@@ -305,7 +320,7 @@ def retrieve(self, request, path):
         return Response(data=data)
 
 
-class UploadView(ViewSet, PackageUploadMixin):
+class UploadView(PackageUploadMixin, ViewSet):
     """View for the `/legacy` upload endpoint."""
 
     @extend_schema(request=PackageUploadSerializer,

pulp_python/app/serializers.py

@@ -5,6 +5,7 @@
 
 from pulpcore.plugin import models as core_models
 from pulpcore.plugin import serializers as core_serializers
+from pulpcore.plugin.util import get_domain
 
 from pulp_python.app import models as python_models
 from pulp_python.app.utils import get_project_metadata_from_artifact, parse_project_metadata
@@ -56,6 +57,8 @@ class PythonDistributionSerializer(core_serializers.DistributionSerializer):
 
     def get_base_url(self, obj):
         """Gets the base url."""
+        if settings.DOMAIN_ENABLED:
+            return f"{settings.PYPI_API_HOSTNAME}/pypi/{get_domain().name}/{obj.base_path}/"
         return f"{settings.PYPI_API_HOSTNAME}/pypi/{obj.base_path}/"
 
     class Meta:
@@ -232,13 +235,17 @@ def deferred_validate(self, data):
         _data['version'] = metadata.version
         _data['filename'] = filename
         _data['sha256'] = artifact.sha256
+        data["pulp_domain_id"] = artifact.pulp_domain_id
+        data["_pulp_domain_id"] = artifact.pulp_domain_id
 
         data.update(_data)
 
         return data
 
     def retrieve(self, validated_data):
-        content = python_models.PythonPackageContent.objects.filter(sha256=validated_data["sha256"])
+        content = python_models.PythonPackageContent.objects.filter(
+            sha256=validated_data["sha256"], _pulp_domain=get_domain()
+        )
         return content.first()
 
     class Meta:

pulp_python/app/tasks/publish.py

@@ -6,6 +6,7 @@
 from packaging.utils import canonicalize_name
 
 from pulpcore.plugin import models
+from pulpcore.plugin.util import get_domain
 
 from pulp_python.app import models as python_models
 from pulp_python.app.utils import write_simple_index, write_simple_detail
@@ -49,11 +50,12 @@ def write_simple_api(publication):
         publication (pulpcore.plugin.models.Publication): A publication to generate metadata for
 
     """
+    domain = get_domain()
     simple_dir = 'simple/'
     os.mkdir(simple_dir)
     project_names = (
         python_models.PythonPackageContent.objects.filter(
-            pk__in=publication.repository_version.content
+            pk__in=publication.repository_version.content, _pulp_domain=domain
         )
         .order_by('name')
         .values_list('name', flat=True)
@@ -76,7 +78,7 @@ def write_simple_api(publication):
         return
 
     packages = python_models.PythonPackageContent.objects.filter(
-        pk__in=publication.repository_version.content
+        pk__in=publication.repository_version.content, _pulp_domain=domain
     )
     releases = packages.order_by("name").values("name", "filename", "sha256")