Build docs with docs deps and make (#956)

* Build docs with docs deps and make * Update CI for publishing docs * Build docs only when docs change Signed-off-by: Mike McKiernan <[email protected]>
NVIDIA · Jan 28, 2025 · 71bcd2d · 71bcd2d
1 parent 715e88b
commit 71bcd2d
Show file tree

Hide file tree

Showing 15 changed files with 2,772 additions and 1,878 deletions.
diff --git a/.github/workflows/docs-build.yaml b/.github/workflows/docs-build.yaml
@@ -0,0 +1,147 @@
+name: docs-build
+
+on:
+    pull_request:
+        branches: [develop]
+        types: [opened, synchronize]
+        paths:
+            - "docs/**"
+            - "**/*.md"
+            - "**/*.rst"
+    push:
+        branches: [develop]
+        tags:
+            - docs-v*
+            - v*
+    workflow_dispatch:
+
+defaults:
+    run:
+        shell: bash
+
+env:
+    PYTHON_VERSION: "3.10"
+    POETRY_VERSION: "1.8.2"
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    build-docs:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Set up Python ${{ env.PYTHON_VERSION }}
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ env.PYTHON_VERSION }}
+            - name: Bootstrap poetry (Linux and macOS)
+              run: |
+                  curl -sSL https://install.python-poetry.org | POETRY_VERSION=${{ env.POETRY_VERSION }} python -
+            - name: Update PATH
+              run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+            - name: Configure poetry
+              run: poetry config virtualenvs.in-project true
+            - name: Install dependencies
+              run: poetry install --only docs
+            - name: Build documentation
+              run: make docs
+            - name: Delete unnecessary files
+              run: |
+                  sudo find _build -name .doctrees -prune -exec rm -rf {} \;
+                  sudo find _build -name .buildinfo -exec rm {} \;
+            - name: Upload HTML
+              uses: actions/upload-artifact@v4
+              with:
+                  name: html-build-artifact
+                  path: _build/docs
+                  if-no-files-found: error
+                  retention-days: 1
+            - name: Store PR information
+              if: github.event_name == 'pull_request'
+              run: |
+                  mkdir ./pr
+                  echo ${{ github.event.number }}              > ./pr/pr.txt
+                  echo ${{ github.event.pull_request.merged }} > ./pr/merged.txt
+                  echo ${{ github.event.action }}              > ./pr/action.txt
+            - name: Upload PR information
+              if: github.event_name == 'pull_request'
+              uses: actions/upload-artifact@v4
+              with:
+                  name: pr
+                  path: pr/
+
+    store-html:
+        if: github.event_name == 'push'
+        needs: [build-docs]
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  ref: "gh-pages"
+            - name: Initialize Git configuration
+              run: |
+                  git config user.name docs-build
+                  git config user.email [email protected]
+            - name: Download artifacts
+              uses: actions/download-artifact@v4
+              with:
+                  name: html-build-artifact
+            - name: Copy HTML directories
+              run: |
+                  ls -asl
+                  for i in `ls -d *`
+                  do
+                    echo "Git adding ${i}"
+                    git add "${i}"
+                  done
+            - name: Check or create dot-no-jekyll file
+              run: |
+                  if [ -f ".nojekyll" ]; then
+                    echo "The dot-no-jekyll file already exists."
+                    exit 0
+                  fi
+                  touch .nojekyll
+                  git add .nojekyll
+            - name: Check or create redirect page
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: |
+                  resp=$(grep 'http-equiv="refresh"' index.html 2>/dev/null) || true
+                  if [ -n "${resp}" ]; then
+                    echo "The redirect file already exists."
+                    exit 0
+                  fi
+                  # If any of these commands fail, fail the build.
+                  def_branch=$(gh api "repos/${GITHUB_REPOSITORY}" --jq ".default_branch")
+                  html_url=$(gh api "repos/${GITHUB_REPOSITORY}/pages" --jq ".html_url")
+                  # Beware ugly quotation mark avoidance in the foll lines.
+                  echo '<!DOCTYPE html>'                                                                         > index.html
+                  echo '<html>'                                                                                 >> index.html
+                  echo '  <head>'                                                                               >> index.html
+                  echo '    <title>Redirect to documentation</title>'                                           >> index.html
+                  echo '    <meta charset="utf-8">'                                                             >> index.html
+                  echo '    <meta http=equiv="refresh" content="3; URL='${html_url}${def_branch}'/index.html">' >> index.html
+                  echo '    <link rel="canonical" href="'${html_url}${def_branch}'/index.html">'                >> index.html
+                  echo '    <script language="javascript">'                                                     >> index.html
+                  echo '      function redirect() {'                                                            >> index.html
+                  echo '        window.location.assign("'${html_url}${def_branch}'/index.html")'                >> index.html
+                  echo '      }'                                                                                >> index.html
+                  echo '    </script>'                                                                          >> index.html
+                  echo '  </head>'                                                                              >> index.html
+                  echo '  <body onload="redirect()">'                                                           >> index.html
+                  echo '    <p>Please follow the link to the <a href="'${html_url}${def_branch}'/index.html">'  >> index.html
+                  echo      ${def_branch}'</a> branch documentation.</p>'                                       >> index.html
+                  echo '  </body>'                                                                              >> index.html
+                  echo '</html>'                                                                                >> index.html
+                  git add index.html
+            - name: Commit changes to the GitHub Pages branch
+              run: |
+                  git status
+                  if git commit -m 'Pushing changes to GitHub Pages.'; then
+                    git push -f
+                  else
+                   echo "Nothing changed."
+                  fi
diff --git a/.github/workflows/docs-preview-pr.yaml b/.github/workflows/docs-preview-pr.yaml
@@ -0,0 +1,18 @@
+name: docs-preview-pr
+
+on:
+    workflow_run:
+        workflows: [docs-build]
+        types: [completed]
+        branches-ignore: [develop]
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+    cancel-in-progress: true
+
+env:
+    WF_ID: ${{ github.event.workflow_run.id }}
+
+jobs:
+    preview:
+        uses: nvidia-merlin/.github/.github/workflows/docs-preview-pr-common.yaml@main
diff --git a/.github/workflows/docs-remove-stale-reviews.yaml b/.github/workflows/docs-remove-stale-reviews.yaml
@@ -0,0 +1,11 @@
+name: docs-remove-stale-reviews
+
+on:
+    schedule:
+        # 42 minutes after 0:00 UTC on Sundays
+        - cron: "42 0 * * 0"
+    workflow_dispatch:
+
+jobs:
+    remove:
+        uses: nvidia-merlin/.github/.github/workflows/docs-remove-stale-reviews-common.yaml@main
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -6,89 +6,93 @@
 # Note that environment variables can be set in several places
 # See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
 stages:
-  - tests
-  - build
-  - docker-test
+    - tests
+    - build
+    - docker-test
+    - deploy
 sast:
-  stage: tests
+    stage: tests
 include:
-  - template: Security/SAST.gitlab-ci.yml
+    - template: Security/SAST.gitlab-ci.yml
+    - project: "sw-gpu-doctools/docs-ci"
+      ref: main
+      file: "nemo-guardrails-toolkit/docs-ci.yaml"
 
 variables:
-  PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
-  POETRY_VERSION: "1.8.2"
-  IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
-  LATEST_TAG: $CI_REGISTRY_IMAGE:latest
+    PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.cache/pip"
+    POETRY_VERSION: "1.8.2"
+    IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
+    LATEST_TAG: $CI_REGISTRY_IMAGE:latest
 
 cache:
-  key: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}"
-  paths:
-    - .venv/
-    - .cache/pip
-    - poetry.lock
-    - pyproject.toml
+    key: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}"
+    paths:
+        - .venv/
+        - .cache/pip
+        - poetry.lock
+        - pyproject.toml
 
 # Jobs templates
 
 .install-deps-template: &install-deps
-  before_script:
-    - pip install poetry==$POETRY_VERSION
-    - poetry --version
-    - poetry config virtualenvs.in-project true
-    - poetry install --extras all --with dev
+    before_script:
+        - pip install poetry==$POETRY_VERSION
+        - poetry --version
+        - poetry config virtualenvs.in-project true
+        - poetry install --extras all --with dev
 
 .test-template: &test
-  <<: *install-deps
-  stage: tests
-  coverage: /(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
-  script: make test
+    <<: *install-deps
+    stage: tests
+    coverage: /(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
+    script: make test
 
 # Tests jobs
 
 python3.9:
-  <<: *test
-  image: python:3.9
+    <<: *test
+    image: python:3.9
 
 python3.10:
-  <<: *test
-  image: python:3.10
+    <<: *test
+    image: python:3.10
 
 python3.11:
-  <<: *test
-  image: python:3.11
+    <<: *test
+    image: python:3.11
 
 # Build job
 build:
-  stage: build
-  image: docker:19.03.12
-  services:
-    - docker:19.03.12-dind
-  variables:
-    DOCKER_DRIVER: overlay2
-    DOCKER_TLS_CERTDIR: "/certs"
-  before_script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-  script:
-    - docker build -t $IMAGE_TAG -f ./qa/Dockerfile.qa .
-    - docker push $IMAGE_TAG
-  only:
-    - tags
-  tags:
-    - gitlab-runner-bignlp-api
+    stage: build
+    image: docker:19.03.12
+    services:
+        - docker:19.03.12-dind
+    variables:
+        DOCKER_DRIVER: overlay2
+        DOCKER_TLS_CERTDIR: "/certs"
+    before_script:
+        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    script:
+        - docker build -t $IMAGE_TAG -f ./qa/Dockerfile.qa .
+        - docker push $IMAGE_TAG
+    only:
+        - tags
+    tags:
+        - gitlab-runner-bignlp-api
 
 # Docker test job
 docker-test:
-  stage: docker-test
-  image: docker:19.03.12
-  services:
-    - docker:19.03.12-dind
-  before_script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-  script:
-    - docker pull $IMAGE_TAG
-    - docker run --rm $IMAGE_TAG pytest || (docker rmi $IMAGE_TAG && exit 1)
-  only:
-    - tags
+    stage: docker-test
+    image: docker:19.03.12
+    services:
+        - docker:19.03.12-dind
+    before_script:
+        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    script:
+        - docker pull $IMAGE_TAG
+        - docker run --rm $IMAGE_TAG pytest || (docker rmi $IMAGE_TAG && exit 1)
+    only:
+        - tags
 
-  tags:
-    - gitlab-runner-bignlp-api
+    tags:
+        - gitlab-runner-bignlp-api
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -135,7 +135,7 @@ To get started quickly, follow the steps below.
   Ensure you have `poetry` installed:
 
    ```bash
-   poetry -- version
+   poetry --version
    ```
 
 6. Install the dev dependencies:
@@ -144,7 +144,8 @@ To get started quickly, follow the steps below.
    poetry install --with dev
    ```
 
-   This will install pre-commit, pytest, and other development tools.
+   The preceding command installs pre-commit, pytest, and other development tools.
+   Specify `--with dev,docs` to add the dependencies for building the documentation.
 
 7. If needed, you can install extra dependencies as below:
 
@@ -159,7 +160,6 @@ To get started quickly, follow the steps below.
 
     ```bash
     poetry install --all-extras
-
     ```
 
 > **Note**: `dev` is not part of the extras but it is an optional dependency group, so you need to install it as instructed above.
@@ -168,7 +168,6 @@ To get started quickly, follow the steps below.
 
    ```
    pre-commit install
-
    ```
 
    This will ensure that the pre-commit checks, including Black, are run before each commit.

diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: all test tests test_watch test_coverage test_profile pre_commit help
+.PHONY: all test tests test_watch test_coverage test_profile docs pre_commit help
 
 # Default target executed when no specific target is provided to make.
 all: help
@@ -21,6 +21,9 @@ test_coverage:
 test_profile:
 	poetry run pytest -vv tests/ --profile-svg
 
+docs:
+	poetry run sphinx-build -b html docs _build/docs
+
 pre_commit:
 	pre-commit install
 	pre-commit run --all-files
@@ -35,4 +38,5 @@ help:
 	@echo 'test TEST_FILE=<test_file>   - run all tests in given file'
 	@echo 'test_watch                   - run unit tests in watch mode'
 	@echo 'test_coverage                - run unit tests with coverage'
+	@echo 'docs                         - build docs, if you installed the docs dependencies'
 	@echo 'pre_commit                   - run pre-commit hooks'