DOC/BUILD add ability for conf to skip whole sections

Author: jklymak

.gitignore

@@ -82,6 +82,7 @@ examples/tests/*
 !examples/tests/backend_driver_sgskip.py
 result_images
 doc/_static/constrained_layout*.png
+doc/.mpl_skip_subdirs.yaml
 
 # Nose/Pytest generated files #
 ###############################

doc/Makefile

@@ -33,6 +33,13 @@ html-noplot:
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
+# This will skip the subdirectories listed in .mpl_skip_subdirs.yaml If
+# this file does not exist, one will be created for you.  This option useful
+# to quickly build parts of the docs, but the resulting build will not
+# have all the crosslinks etc.
+html-skip-subdirs:
+	$(SPHINXBUILD) -D skip_sub_dirs=1 -b html $(SOURCEDIR) $(BUILDDIR)/html $(SPHINXOPTS) $(O)
+
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile

doc/conf.py

@@ -19,6 +19,7 @@
 import sys
 from urllib.parse import urlsplit, urlunsplit
 import warnings
+import yaml
 
 import matplotlib
 
@@ -34,6 +35,42 @@
 # are we running circle CI?
 CIRCLECI = 'CIRCLECI' in os.environ
 
+
+def _parse_skip_subdirs_file():
+    """
+    Read .mpl_skip_subdirs.yaml for subdirectories to not
+    build if we do `make html-skip-subdirs`.  Subdirectories
+    are relative to the toplevel directory.  Note that you
+    cannot skip 'users' as it contains the table of contents,
+    but you can skip subdirectories of 'users'.  Doing this
+    can make partial builds very fast.
+    """
+    default_skip_subdirs = ['users/prev_whats_new/*', 'api/*', 'gallery/*',
+                            'tutorials/*', 'plot_types/*', 'devel/*']
+    try:
+        with open(".mpl_skip_subdirs.yaml", 'r') as fin:
+            print('Reading subdirectories to skip from',
+                  '.mpl_skip_subdirs.yaml')
+            out = yaml.full_load(fin)
+        return out['skip_subdirs']
+    except FileNotFoundError:
+        # make a default:
+        with open(".mpl_skip_subdirs.yaml", 'w') as fout:
+            yamldict = {'skip_subdirs': default_skip_subdirs,
+                        'comment': 'For use with make html-skip-subdirs'}
+            yaml.dump(yamldict, fout)
+        print('Skipping subdirectories, but .mpl_skip_subdirs.yaml',
+              'not found so creating a default one. Edit this file',
+              'to customize which directories are included in build.')
+
+        return default_skip_subdirs
+
+
+skip_subdirs = []
+# triggered via make html-skip-subdirs
+if 'skip_sub_dirs=1' in sys.argv:
+    skip_subdirs = _parse_skip_subdirs_file()
+
 # Parse year using SOURCE_DATE_EPOCH, falling back to current time.
 # https://reproducible-builds.org/specs/source-date-epoch/
 sourceyear = datetime.utcfromtimestamp(
@@ -80,9 +117,11 @@
 ]
 
 exclude_patterns = [
-    'api/prev_api_changes/api_changes_*/*',
+    'api/prev_api_changes/api_changes_*/*'
 ]
 
+exclude_patterns += skip_subdirs
+
 
 def _check_dependencies():
     names = {
@@ -174,15 +213,20 @@ def matplotlib_reduced_latex_scraper(block, block_vars, gallery_conf,
         gallery_conf['image_srcset'] = []
     return matplotlib_scraper(block, block_vars, gallery_conf, **kwargs)
 
+gallery_dirs = [f'{ed}' for ed in ['gallery', 'tutorials', 'plot_types']
+                if f'{ed}/*' not in skip_subdirs]
+
+example_dirs = [f'../{gd}'.replace('gallery', 'examples') for gd in
+                gallery_dirs]
 
 sphinx_gallery_conf = {
     'backreferences_dir': Path('api') / Path('_as_gen'),
     # Compression is a significant effort that we skip for local and CI builds.
     'compress_images': ('thumbnails', 'images') if is_release_build else (),
     'doc_module': ('matplotlib', 'mpl_toolkits'),
-    'examples_dirs': ['../examples', '../tutorials', '../plot_types'],
+    'examples_dirs': example_dirs,
     'filename_pattern': '^((?!sgskip).)*$',
-    'gallery_dirs': ['gallery', 'tutorials', 'plot_types'],
+    'gallery_dirs': gallery_dirs,
     'image_scrapers': (matplotlib_reduced_latex_scraper, ),
     'image_srcset': ["2x"],
     'junit': '../test-results/sphinx-gallery/junit.xml' if CIRCLECI else '',
@@ -711,5 +755,6 @@ def setup(app):
         bld_type = 'dev'
     else:
         bld_type = 'rel'
+    app.add_config_value('skip_sub_dirs', 0, '')
     app.add_config_value('releaselevel', bld_type, 'env')
     app.connect('html-page-context', add_html_cache_busting, priority=1000)

doc/devel/documenting_mpl.rst

@@ -71,6 +71,12 @@ Other useful invocations include
    # save time.
    make html-noplot
 
+   # Build the html documentation, but skip specific subdirectories.  If a gallery
+   # directory is skipped, the gallery images are not generated.  The first
+   # time this is run, it creates ``.mpl_skip_subdirs.yaml`` which can be edited
+   # to add or remove subdirectories
+   make html-skip-subdirs
+
    # Delete built files.  May help if you get errors about missing paths or
    # broken links.
    make clean

doc/make.bat

@@ -29,6 +29,7 @@ if errorlevel 9009 (
 
 if "%1" == "" goto help
 if "%1" == "html-noplot" goto html-noplot
+if "%1" == "html-skip-subdirs" goto html-skip-subdirs
 if "%1" == "show" goto show
 
 %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
@@ -52,6 +53,11 @@ goto end
 %SPHINXBUILD% -M html %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -D plot_gallery=0
 goto end
 
+:html-skip-subdirs
+%SPHINXBUILD% -M html %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -D skip_sub_dirs=1
+goto end
+
+
 :show
 python -m webbrowser -t "%~dp0\build\html\index.html"
 

environment.yml

@@ -33,6 +33,7 @@ dependencies:
   - numpydoc>=0.8
   - packaging
   - pydata-sphinx-theme
+  - pyyaml
   - sphinx>=1.8.1,!=2.0.0
   - sphinx-copybutton
   - sphinx-gallery>=0.10

requirements/doc/doc-requirements.txt

@@ -15,6 +15,7 @@ numpydoc>=1.0
 packaging>=20
 pydata-sphinx-theme>=0.12.0
 mpl-sphinx-theme
+pyyaml
 sphinxcontrib-svg2pdfconverter>=1.1.0
 sphinx-gallery>=0.10
 sphinx-copybutton