Add support for subpackages (like pulp_file)

Author: pedro-psb

README.md

@@ -12,10 +12,10 @@ Python Package to help aggregating Pulp's multirepo ecosystem into a unified doc
 
 This packages is:
 
-- A `mkdocs-macros-plugin` [pluget](https://mkdocs-macros-plugin.readthedocs.io/en/latest/pluglets/). [relevant-code]()
-- A repository for common doc website asset. [relevant-code](https://github.com/pedro-psb/pulp-docs/tree/main/src/pulp_docs/docs)
+- A `mkdocs-macros-plugin` [pluget](https://mkdocs-macros-plugin.readthedocs.io/en/latest/pluglets/).
+- A repository for common doc website asset.
 - A centralized entrypoint for installing doc-related packages/tooling. (via its own requirements)
-- A CLI for doc-related tasks, like serving and building. [relevant-code](https://github.com/pedro-psb/pulp-docs/blob/main/src/pulp_docs/cli.py)
+- A CLI for doc-related tasks, like serving and building.
 
 The idea is that each repository should install `pulp-docs` and imediatelly be able run the unified website server.
 Also, this should be used for the production build.
@@ -24,7 +24,7 @@ Also, this should be used for the production build.
 
 Through a `mkdocs-macro-plugin` hook (called in early stages of mkdocs processing), we inject the following steps:
 
-1. Read `repolist.yml` packaged with `pulp-docs` to know which repos/urls to use
+1. Read [`repolist.yml`](https://github.com/pedro-psb/pulp-docs/blob/main/src/pulp_docs/data/repolist.yml) packaged with `pulp-docs` to know which repos/urls to use
 1. Download/Move all source code required to dir under `tempfile.gettempdir()`
     - Uses `../{repo}` if available OR
     - Uses existing cached `{tmpdir}/{repo}` if available OR
@@ -39,13 +39,13 @@ And thats it, the magic is done.
 Recommended way for daily usage:
 
 ```bash
-$ pipx install git+https://github.com/pedro-psb/pulp-docs --include-deps
-$ pulp-docs serve
+pipx install git+https://github.com/pedro-psb/pulp-docs --include-deps
+pulp-docs serve
 ```
 
 For development, use your prefered method!
 
-## How to share inter-repo "unofficial" work
+## How to override `repolist.yml`
 
 If you want to share work you are doing in muliple forks, you can share a custom `repolist.yml` which points to your forks.
 

src/pulp_docs/constants.py

@@ -1,5 +1,6 @@
 ADMIN_NAME = "admin"
 USER_NAME = "user"
+RESTAPI_URL_TEMPLATE = "https://docs.pulpproject.org/{}/restapi.html"
 
 DISPLAY_NAMES = {
     "guides": "How-to Guides",

src/pulp_docs/data/repolist.yml

@@ -1,4 +1,5 @@
 meta:
+  version: 1
   rest_api_template: https://docs.pulpproject.org/{}/restapi.html  
 
 repos:
@@ -24,10 +25,6 @@ repos:
       owner: pulp
       title: Ansible
       branch: main
-    - name: pulp-certguard
-      owner: pulp
-      title: Certguard
-      branch: main
     - name: pulp_container
       owner: pulp
       title: Container
@@ -44,10 +41,13 @@ repos:
       owner: pulp
       title: RPM
       branch: main
-    # - name: pulp_file
-    #   nested_under: pulp/pulpcore/pulp_file
-    #   title: File
-    #   branch: main
+    # subpackages
+    - name: pulp_certguard
+      title: Certguard
+      subpackage_of: pulpcore
+    - name: pulp_file
+      title: File
+      subpackage_of: pulpcore
   other:
     - name: pulp-oci-images
       owner: pulp

src/pulp_docs/mkdocs_macros.py

@@ -24,8 +24,9 @@
 import rich
 
 from pulp_docs.cli import Config
+from pulp_docs.constants import RESTAPI_URL_TEMPLATE
 from pulp_docs.navigation import get_navigation
-from pulp_docs.repository import Repos
+from pulp_docs.repository import Repo, Repos, SubPackage
 
 # the name of the docs in the source repositories
 SRC_DOCS_DIRNAME = "staging_docs"
@@ -91,47 +92,23 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config):
 
     for repo in repos.all:
         start = time.perf_counter()
-        # if repo.name == "pulp-docs":
-        # breakpoint()
-        # 1. Download repo (copy locally or fetch from GH)
-        this_src_dir = repo_sources / repo.name
-        repo.download(dest_dir=this_src_dir, clear_cache=config.clear_cache)
-
-        # 2. Isolate docs dir from codebase (makes mkdocs happy)
+        # handle subpcakges nested under repositories
         this_docs_dir = repo_docs / repo.name
-        log.info(
-            "Moving doc files:\nfrom '{}'\nto '{}'".format(this_src_dir, this_docs_dir)
-        )
+        if not isinstance(repo, SubPackage):
+            this_src_dir = repo_sources / repo.name
+            repo.download(dest_dir=this_src_dir, clear_cache=config.clear_cache)
+        else:
+            this_src_dir = repo_sources / repo.subpackage_of / repo.name
 
-        try:
-            shutil.copytree(this_src_dir / SRC_DOCS_DIRNAME, this_docs_dir / "docs")
-        except FileNotFoundError:
-            Path(this_docs_dir / "docs").mkdir(parents=True)
-            repo.status.has_staging_docs = False
-        try:
-            shutil.copy(this_src_dir / "CHANGELOG.md", this_docs_dir / "CHANGELOG.md")
-        except FileNotFoundError:
-            repo.status.has_changelog = False
-
-        try:
-            shutil.copy(this_src_dir / "README.md", this_docs_dir / "README.md")
-        except FileNotFoundError:
-            repo.status.has_readme = False
-
-        # 3. Generate REST Api pages (workaround)
+        # install and post-process
+        _install_doc_files(this_src_dir, this_docs_dir, repo)
         if repo.type == "content":
-            log.info("Generating REST_API page")
-            rest_api_page = this_docs_dir / "docs" / "rest_api.md"
-            rest_api_page.touch()
-            md_title = f"# {repo.title} REST Api"
-            md_body = f"[{repo.rest_api_link}]({repo.rest_api_link})"
-            rest_api_page.write_text(f"{md_title}\n\n{md_body}")
+            _generate_rest_api_page(this_docs_dir, repo.name, repo.title)
 
         end = time.perf_counter()
-        duration = end - start
-        log.info(f"{repo.name} completed in {duration:.2} sec")
+        log.info(f"{repo.name} completed in {end - start:.2} sec")
 
-    # Copy template-files (from this plugin) to tmpdir
+    # Copy core-files (shipped with pulp-docs) to tmpdir
     shutil.copy(
         repo_sources / repos.core_repo.name / SRC_DOCS_DIRNAME / "index.md",
         repo_docs / "index.md",
@@ -145,6 +122,37 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config):
     return (repo_docs, repo_sources)
 
 
+def _install_doc_files(src_dir: Path, docs_dir: Path, repo: Repo):
+    """Copy only doc-related files from src_dir to doc_dir"""
+    log.info(f"Moving doc files:\nfrom '{src_dir}'\nto '{docs_dir}'")
+
+    try:
+        shutil.copytree(src_dir / SRC_DOCS_DIRNAME, docs_dir / "docs")
+    except FileNotFoundError:
+        Path(docs_dir / "docs").mkdir(parents=True)
+        repo.status.has_staging_docs = False
+    try:
+        shutil.copy(src_dir / "CHANGELOG.md", docs_dir / "CHANGELOG.md")
+    except FileNotFoundError:
+        repo.status.has_changelog = False
+
+    try:
+        shutil.copy(src_dir / "README.md", docs_dir / "README.md")
+    except FileNotFoundError:
+        repo.status.has_readme = False
+
+
+def _generate_rest_api_page(docs_dir: Path, repo_name: str, repo_title: str):
+    """Create page that contain a link to the rest api, based on the project url template"""
+    log.info("Generating REST_API page")
+    rest_api_page = docs_dir / "docs" / "rest_api.md"
+    rest_api_page.touch()
+    restapi_url = RESTAPI_URL_TEMPLATE.format(repo_name)
+    md_title = f"# {repo_title} REST Api"
+    md_body = f"[{restapi_url}]({restapi_url})"
+    rest_api_page.write_text(f"{md_title}\n\n{md_body}")
+
+
 def print_user_repo(repos: Repos, config: Config):
     """Emit report  about local checkout being used or warn if none."""
     print("*" * 79)

src/pulp_docs/repository.py

@@ -12,6 +12,7 @@
 import tarfile
 import tempfile
 import typing as t
+from collections import ChainMap, defaultdict
 from dataclasses import dataclass, field
 from io import BytesIO
 from pathlib import Path
@@ -25,7 +26,6 @@
 
 FIXTURE_WORKDIR = Path("tests/fixtures").absolute()
 DOWNLOAD_CACHE_DIR = Path(tempfile.gettempdir()) / "repo_downloads"
-RESTAPI_TEMPLATE = "https://docs.pulpproject.org/{}/restapi.html"
 
 
 # @dataclass # raising errors in py311/312
@@ -61,6 +61,7 @@ class Repo:
     branch: str = "main"
     branch_in_use: t.Optional[str] = None
     local_basepath: t.Optional[Path] = None
+    subpackages: t.Optional[t.List] = None
     status: RepoStatus = field(default_factory=lambda: RepoStatus())
     type: t.Optional[str] = None
 
@@ -190,6 +191,20 @@ def download_from_gh_latest(dest_dir: Path, owner: str, name: str):
     return latest_release_tar_url
 
 
+@dataclass
+class SubPackage:
+    """A package that lives under another Repo."""
+
+    name: str
+    title: str
+    subpackage_of: str
+    type: t.Optional[str] = None
+    status: RepoStatus = field(default_factory=lambda: RepoStatus())
+    local_basepath = None
+    branch_in_use = ""
+    branch = ""
+
+
 @dataclass
 class Repos:
     """A collection of Repos"""
@@ -216,7 +231,25 @@ def get(self, repo_name: str) -> t.Optional[Repo]:
 
     @property
     def all(self):
-        return [self.core_repo] + self.content_repos + self.other_repos
+        """The set of repositories and subpackages"""
+        repos = [self.core_repo] + self.content_repos + self.other_repos
+        subpackages = []
+        for repo in repos:
+            if repo.subpackages:
+                subpackages.extend(repo.subpackages)
+        return repos + subpackages
+
+    def get_repos(self, type="all"):
+        """Get a set of repositories and subpackages by type."""
+        if type == "all":
+            return self.all
+
+        repos_and_pkgs = []
+        for repo in self.all:
+            if getattr(repo, "subpackages", None):
+                repos_and_pkgs.extend(repo.subpackages)
+
+        return [repo for repo in repos_and_pkgs if repo.type == type]
 
     @classmethod
     def from_yaml(cls, path: str):
@@ -242,14 +275,34 @@ def from_yaml(cls, path: str):
             raise ValueError("File does not exist:", file)
         log.info(f"repofile={str(file.absolute())}")
 
+        # Create Repo objects from yaml data
+        repos: t.Dict[str, t.List] = {}
+        nested_packages: t.Dict[str, t.List[SubPackage]] = {}
         with open(file, "r") as f:
             data = yaml.load(f, Loader=yaml.SafeLoader)
-        repos = data["repos"]
-        core_repo = Repo(**repos["core"][0], type="core")
-        content_repos = [Repo(**repo, type="content") for repo in repos["content"]]
-        other_repos = [Repo(**repo, type="other") for repo in repos["other"]]
+            for repo_type in ("core", "content", "other"):
+                repos[repo_type] = []
+                for repo in data["repos"][repo_type]:
+                    # Collect nested packages
+                    if parent_package := repo.get("subpackage_of", None):
+                        nested_packages.setdefault(parent_package, []).append(
+                            SubPackage(**repo, type=repo_type)
+                        )
+                        continue
+                    # Create regular packages
+                    repos[repo_type].append(Repo(**repo, type=repo_type))
+
+        # Update Repo objects that contain subpackages
+        for parent_repo_name, subpackages_list in nested_packages.items():
+            flat_repos = repos["core"] + repos["content"] + repos["other"]
+            for repo in flat_repos:
+                if repo.name == parent_repo_name:
+                    repo.subpackages = subpackages_list
+
         return Repos(
-            core_repo=core_repo, content_repos=content_repos, other_repos=other_repos
+            core_repo=repos["core"][0],
+            content_repos=repos["content"],
+            other_repos=repos["other"],
         )
 
     @classmethod

src/pulp_docs/utils/aggregation.py

@@ -94,7 +94,7 @@ def repo_grouping(
             selected_repos = self.repos.all
         else:
             for repo_name in repo_types:
-                selected_repos.extend(getattr(self.repos, f"{repo_name}_repos"))
+                selected_repos.extend(self.repos.get_repos(type=type))
 
         # Dont expand content-types
         if not _expand_content_types: