Add api-generation support w/ swagger-ui-tag

closes: #40

Author: pedro-psb

Diff for: pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
     "mkdocstrings-python>=1.9.1",
     "mkdocs-macros-plugin",
     "mkdocs-site-urls",
+    "mkdocs-swagger-ui-tag",
     "importlib_resources",
     "httpx",
     "rich",

Diff for: src/pulp_docs/data/mkdocs.yml

@@ -61,6 +61,11 @@ plugins:
             separate_signature: false
             members_order: "source"
             merge_init_into_class: true
+  - swagger-ui-tag:
+      docExpansion: none
+      filter_filenames: ["rest_api"]
+
+
 extra:
   social:
     - icon: fontawesome/brands/github-alt

Diff for: src/pulp_docs/mkdocs_macros.py

@@ -17,12 +17,12 @@
 
 import json
 import logging
-import os
 import shutil
 import tempfile
 import time
 from pathlib import Path
 
+import httpx
 import rich
 
 from pulp_docs.cli import Config
@@ -89,6 +89,7 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config):
     # Download/copy source code to tmpdir
     repo_sources = TMPDIR / "repo_sources"
     repo_docs = TMPDIR / "repo_docs"
+    api_src_dir = TMPDIR / "api_json"
     shutil.rmtree(repo_sources, ignore_errors=True)
     shutil.rmtree(repo_docs, ignore_errors=True)
 
@@ -102,10 +103,13 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config):
         else:
             this_src_dir = repo_sources / repo_or_pkg.subpackage_of / repo_or_pkg.name
 
+        # restapi
+        if repo_or_pkg.type in ("content", "core"):
+            _download_api_json(api_src_dir, repo_or_pkg.name)
+            _generate_rest_api_page(this_src_dir, repo_or_pkg.name, repo_or_pkg.title)
+
         # install and post-process
-        _place_doc_files(this_src_dir, this_docs_dir, repo_or_pkg)
-        if repo_or_pkg.type == "content":
-            _generate_rest_api_page(this_docs_dir, repo_or_pkg.name, repo_or_pkg.title)
+        _place_doc_files(this_src_dir, this_docs_dir, repo_or_pkg, api_src_dir)
 
         end = time.perf_counter()
         log.info(f"{repo_or_pkg.name} completed in {end - start:.2} sec")
@@ -124,7 +128,27 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config):
     return (repo_docs, repo_sources)
 
 
-def _place_doc_files(src_dir: Path, docs_dir: Path, repo: Repo):
+def _download_api_json(api_dir: Path, repo_name: str):
+    api_json_path = api_dir / f"{repo_name}/api.json"
+    if api_json_path.exists():
+        log.info(f"{repo_name} api.json already downloaded.")
+        return
+
+    log.info(f"Downloading api.json for {repo_name}")
+    api_url_1 = "https://docs.pulpproject.org/{repo_name}/api.json"
+    api_url_2 = "https://docs.pulpproject.org/{repo_name}/_static/api.json"
+    response = httpx.get(api_url_1.format(repo_name=repo_name))
+    if response.is_error:
+        response = httpx.get(api_url_2.format(repo_name=repo_name))
+    if response.is_error:
+        raise Exception("Couldnt get rest api page")
+
+    api_json_path.parent.mkdir(parents=True, exist_ok=True)
+    api_json_path.write_text(json.dumps(response.json()))
+    log.info("Done.")
+
+
+def _place_doc_files(src_dir: Path, docs_dir: Path, repo: Repo, api_src_dir: Path):
     """
     Copy only doc-related files from src_dir to doc_dir.
 
@@ -148,6 +172,11 @@ def _place_doc_files(src_dir: Path, docs_dir: Path, repo: Repo):
         Path(docs_dir / "docs").mkdir(parents=True)
         repo.status.has_staging_docs = False
 
+    # Get restapi
+    if repo.type in ("content", "core"):
+        api_json = api_src_dir / f"{repo.name}/api.json"
+        shutil.copy(api_json, docs_dir / "docs/api.json")
+
     # Get changelog
     repo.status.has_changelog = False
     changes_dir = Path(docs_dir / "changes")
@@ -175,15 +204,24 @@ def _place_doc_files(src_dir: Path, docs_dir: Path, repo: Repo):
     )
 
 
+RESTAPI_TEMPLATE = """\
+---
+hide:
+  - toc
+---
+
+# {repo_title} REST Api
+
+<swagger-ui src="api.json"/>
+"""
+
+
 def _generate_rest_api_page(docs_dir: Path, repo_name: str, repo_title: str):
     """Create page that contain a link to the rest api, based on the project url template"""
     log.info("Generating REST_API page")
-    rest_api_page = docs_dir / "docs" / "rest_api.md"
-    rest_api_page.touch()
-    restapi_url = RESTAPI_URL_TEMPLATE.format(repo_name)
-    md_title = f"# {repo_title} REST Api"
-    md_body = f"[{restapi_url}]({restapi_url})"
-    rest_api_page.write_text(f"{md_title}\n\n{md_body}")
+    rest_api_page = docs_dir / "staging_docs" / "rest_api.md"
+    rest_api_page.parent.mkdir(parents=True, exist_ok=True)
+    rest_api_page.write_text(RESTAPI_TEMPLATE.format(repo_title=repo_title))
 
 
 def print_user_repo(repos: Repos, config: Config):

Diff for: src/pulp_docs/navigation.py

@@ -68,6 +68,7 @@ def grouped_by_persona(tmpdir: Path, repos: Repos):
                 ),
                 f.section(Names.LEARN, f.get_children, "pulpcore/docs/user/learn"),
                 f.section(Names.GUIDES, f.get_children, "pulpcore/docs/user/guides"),
+                {"Rest API": "pulpcore/docs/rest_api.md"}
             ]
         },
         {

Diff for: src/pulp_docs/repository.py

@@ -6,14 +6,14 @@
 
 from __future__ import annotations
 
+import json
 import logging
 import os
 import shutil
 import subprocess
 import tarfile
 import tempfile
 import typing as t
-from collections import ChainMap, defaultdict
 from dataclasses import dataclass, field
 from io import BytesIO
 from pathlib import Path
@@ -116,13 +116,13 @@ def download(self, dest_dir: Path, clear_cache: bool = False) -> str:
         # from remote
         elif not cached_repo.exists():
             log_header = "Downloading from remote"
+            src_copy_path = DOWNLOAD_CACHE_DIR / self.name
             download_from = download_from_gh_main(
-                DOWNLOAD_CACHE_DIR / self.name,
+                src_copy_path,
                 self.owner,
                 self.name,
                 self.branch_in_use,
             )
-            src_copy_path = DOWNLOAD_CACHE_DIR / self.name
 
         # copy from source/cache to pulp-docs workdir
         log.info(f"{log_header}: source={download_from}, copied_from={src_copy_path}")
@@ -133,12 +133,15 @@ def download(self, dest_dir: Path, clear_cache: bool = False) -> str:
             src_copy_path,
             dest_dir,
             ignore=shutil.ignore_patterns(*ignore_patterns),
+            dirs_exist_ok=True
         )
 
         self.status.download_source = str(download_from)
         return self.status.download_source
 
 
+
+
 def download_from_gh_main(dest_dir: Path, owner: str, name: str, branch: str):
     """
     Download repository source-code from main

Diff for: src/pulp_docs/utils/aggregation.py

@@ -137,6 +137,8 @@ def repo_grouping(
                     if _repo_content:
                         content_type_name = Names.get(content_type)
                         repo_nav[content_type_name] = _repo_content  # type: ignore
+                if repo.type == "content" and "docs/user" in template_str:
+                    repo_nav["Rest API"] = f"{repo.name}/docs/rest_api.md"
                 if repo_nav:
                     _nav[repo.title] = repo_nav
         return _nav or ["#"]