Add support for subpackages (like pulp_file)

Author: pedro-psb

README.md

@@ -12,10 +12,10 @@ Python Package to help aggregating Pulp's multirepo ecosystem into a unified doc
 
 This packages is:
 
-- A `mkdocs-macros-plugin` [pluget](https://mkdocs-macros-plugin.readthedocs.io/en/latest/pluglets/). [relevant-code]()
-- A repository for common doc website asset. [relevant-code](https://github.com/pedro-psb/pulp-docs/tree/main/src/pulp_docs/docs)
+- A `mkdocs-macros-plugin` [pluget](https://mkdocs-macros-plugin.readthedocs.io/en/latest/pluglets/).
+- A repository for common doc website asset.
 - A centralized entrypoint for installing doc-related packages/tooling. (via its own requirements)
-- A CLI for doc-related tasks, like serving and building. [relevant-code](https://github.com/pedro-psb/pulp-docs/blob/main/src/pulp_docs/cli.py)
+- A CLI for doc-related tasks, like serving and building.
 
 The idea is that each repository should install `pulp-docs` and imediatelly be able run the unified website server.
 Also, this should be used for the production build.
@@ -24,7 +24,7 @@ Also, this should be used for the production build.
 
 Through a `mkdocs-macro-plugin` hook (called in early stages of mkdocs processing), we inject the following steps:
 
-1. Read `repolist.yml` packaged with `pulp-docs` to know which repos/urls to use
+1. Read [`repolist.yml`](https://github.com/pedro-psb/pulp-docs/blob/main/src/pulp_docs/data/repolist.yml) packaged with `pulp-docs` to know which repos/urls to use
 1. Download/Move all source code required to dir under `tempfile.gettempdir()`
     - Uses `../{repo}` if available OR
     - Uses existing cached `{tmpdir}/{repo}` if available OR
@@ -39,13 +39,13 @@ And thats it, the magic is done.
 Recommended way for daily usage:
 
 ```bash
-$ pipx install git+https://github.com/pedro-psb/pulp-docs --include-deps
-$ pulp-docs serve
+pipx install git+https://github.com/pedro-psb/pulp-docs --include-deps
+pulp-docs serve
 ```
 
 For development, use your prefered method!
 
-## How to share inter-repo "unofficial" work
+## How to override `repolist.yml`
 
 If you want to share work you are doing in muliple forks, you can share a custom `repolist.yml` which points to your forks.
 

src/pulp_docs/data/repolist.yml

@@ -1,4 +1,5 @@
 meta:
+  version: 1
   rest_api_template: https://docs.pulpproject.org/{}/restapi.html  
 
 repos:
@@ -24,10 +25,6 @@ repos:
       owner: pulp
       title: Ansible
       branch: main
-    - name: pulp-certguard
-      owner: pulp
-      title: Certguard
-      branch: main
     - name: pulp_container
       owner: pulp
       title: Container
@@ -44,10 +41,13 @@ repos:
       owner: pulp
       title: RPM
       branch: main
-    # - name: pulp_file
-    #   nested_under: pulp/pulpcore/pulp_file
-    #   title: File
-    #   branch: main
+    # subpackages
+    - name: pulp_certguard
+      title: Certguard
+      subpackage_of: pulpcore
+    - name: pulp_file
+      title: File
+      subpackage_of: pulpcore
   other:
     - name: pulp-oci-images
       owner: pulp

src/pulp_docs/repository.py

@@ -12,6 +12,7 @@
 import tarfile
 import tempfile
 import typing as t
+from collections import ChainMap, defaultdict
 from dataclasses import dataclass, field
 from io import BytesIO
 from pathlib import Path
@@ -61,6 +62,7 @@ class Repo:
     branch: str = "main"
     branch_in_use: t.Optional[str] = None
     local_basepath: t.Optional[Path] = None
+    subpackages: t.Optional[t.List] = None
     status: RepoStatus = field(default_factory=lambda: RepoStatus())
     type: t.Optional[str] = None
 
@@ -190,6 +192,15 @@ def download_from_gh_latest(dest_dir: Path, owner: str, name: str):
     return latest_release_tar_url
 
 
+@dataclass
+class SubPackage:
+    """A package that lives under another Repo."""
+
+    name: str
+    title: str
+    type: t.Optional[str] = None
+
+
 @dataclass
 class Repos:
     """A collection of Repos"""
@@ -242,14 +253,34 @@ def from_yaml(cls, path: str):
             raise ValueError("File does not exist:", file)
         log.info(f"repofile={str(file.absolute())}")
 
+        # Create Repo objects from yaml data
+        repos: t.Dict[str, t.List] = {}
+        nested_packages: t.Dict[str, t.List[SubPackage]] = {}
         with open(file, "r") as f:
             data = yaml.load(f, Loader=yaml.SafeLoader)
-        repos = data["repos"]
-        core_repo = Repo(**repos["core"][0], type="core")
-        content_repos = [Repo(**repo, type="content") for repo in repos["content"]]
-        other_repos = [Repo(**repo, type="other") for repo in repos["other"]]
+            for repo_type in ("core", "content", "other"):
+                repos[repo_type] = []
+                for repo in data["repos"][repo_type]:
+                    # Collect nested packages
+                    if parent_package := repo.pop("subpackage_of", None):
+                        nested_packages.setdefault(parent_package, []).append(
+                            SubPackage(**repo, type=repo_type)
+                        )
+                        continue
+                    # Create regular packages
+                    repos[repo_type].append(Repo(**repo, type=repo_type))
+
+        # Update Repo objects that contain subpackages
+        for parent_repo_name, subpackages_list in nested_packages.items():
+            flat_repos = repos["core"] + repos["content"] + repos["other"]
+            for repo in flat_repos:
+                if repo.name == parent_repo_name:
+                    repo.subpackages = subpackages_list
+
         return Repos(
-            core_repo=core_repo, content_repos=content_repos, other_repos=other_repos
+            core_repo=repos["core"][0],
+            content_repos=repos["content"],
+            other_repos=repos["other"],
         )
 
     @classmethod