doc: Enable Markdown doc compilation (#5048)

The initial commit simply uncomments the Markdown compilation lines from #3849. (This requires the actual Markdown files to be present which are now part of the source code.)

Subsequent commits add fixes of the compilation (missing tools, files, etc.):

* Add compilation of multi-program directories like r.colors, r.mapcalc, and r3.flow
* Generate colortable thumbnails. This makes the script little more general and assumes that HTML will be eventually removed (the thumbnails are generated twice now, once for HTML and once of Markdown).
* Copy barscale and northarrow files for Markdown doc source.
* Presence of spaces vs underscores in key may differ between HTML and Markdown, so always do replacements.
* Do not add dash to visible text for keyword. Additionally, remove underscore just like in the HTML version.
* Add projectionintro and grass_database pages from doc dir.
* Add compilation for GUI, icons, and additional non-Markdown files.
* Add basic port of graphical index (removes broken links and provides some content).
* Strip keywords from whitespace and skip empty ones

The complex r.mapcalc build would fail on the first run attempt presumably because of a raise condition with color table build. This removes it from standard SUBDIR and relies on the extra pre-builld of r.mapcalc dir. However, this fails to do distclean properly and needs to be addressed later.

g.extension now creates md from html on the fly if no md is present. The build of Markdown and HTML doc creates a race condition while getting the metadata though git clone, so the build now checks for presence of grass-addons clone and uses the directory as a cache after a successful clone. The code assumes all directory move errors are only the good case of a cache being created in the meantime. The code also creates two (or more) clones if multiple processes run together (git clone runs twice, but the clash in the cache is avoided).

---------

Co-authored-by: Anna Petrasova <[email protected]>

display/d.barscale/Makefile

@@ -10,11 +10,18 @@ include $(MODULE_TOPDIR)/include/Make/Module.make
 # thumbnail previews
 IMGSRC := $(wildcard thumbnails/*.png)
 IMGDST := $(patsubst thumbnails/%,$(HTMLDIR)/barscales/%,$(IMGSRC))
+MDIMGDST := $(patsubst thumbnails/%,$(MDDIR)/source/barscales/%,$(IMGSRC))
 
-default: cmd $(IMGDST)
+default: cmd $(IMGDST) $(MDIMGDST)
 
 $(HTMLDIR)/barscales/%.png: thumbnails/%.png | $(HTMLDIR)/barscales
 	$(INSTALL_DATA) $< $@
 
+$(MDDIR)/source/barscales/%.png: thumbnails/%.png | $(MDDIR)/source/barscales
+	$(INSTALL_DATA) $< $@
+
 $(HTMLDIR)/barscales: $(HTMLDIR)
 	$(MKDIR) $@
+
+$(MDDIR)/source/barscales: $(MDDIR)
+	$(MKDIR) $@

display/d.graph/Makefile

@@ -7,7 +7,10 @@ DEPENDENCIES = $(DISPLAYDEP) $(SYMBDEP) $(GISDEP)
 
 include $(MODULE_TOPDIR)/include/Make/Module.make
 
-default: cmd $(HTMLDIR)/grass_logo.txt
+default: cmd $(HTMLDIR)/grass_logo.txt $(MDDIR)/source/grass_logo.txt
 
 $(HTMLDIR)/%.txt: %.txt
 	$(INSTALL_DATA) $< $@
+
+$(MDDIR)/source/%.txt: %.txt
+	$(INSTALL_DATA) $< $@

display/d.northarrow/Makefile

@@ -10,11 +10,18 @@ include $(MODULE_TOPDIR)/include/Make/Module.make
 # thumbnail previews
 IMGSRC := $(wildcard thumbnails/*.png)
 IMGDST := $(patsubst thumbnails/%,$(HTMLDIR)/northarrows/%,$(IMGSRC))
+MDIMGDST := $(patsubst thumbnails/%,$(MDDIR)/source/northarrows/%,$(IMGSRC))
 
-default: cmd $(IMGDST)
+default: cmd $(IMGDST) $(MDIMGDST)
 
 $(HTMLDIR)/northarrows/%.png: thumbnails/%.png | $(HTMLDIR)/northarrows
 	$(INSTALL_DATA) $< $@
 
+$(MDDIR)/source/northarrows/%.png: thumbnails/%.png | $(MDDIR)/source/northarrows
+	$(INSTALL_DATA) $< $@
+
 $(HTMLDIR)/northarrows: $(HTMLDIR)
 	$(MKDIR) $@
+
+$(MDDIR)/source/northarrows: $(MDDIR)
+	$(MKDIR) $@

doc/Makefile

@@ -3,5 +3,8 @@ MODULE_TOPDIR = ..
 include $(MODULE_TOPDIR)/include/Make/Other.make
 
 FILES := $(wildcard *.html)
+# So far, we disntinguished user and contributor documentation here by
+# extension. This is no longer possible with Markdown.
+MDFILES := grass_database.md projectionintro.md
 
-default: $(patsubst %,$(HTMLDIR)/%,$(FILES))
+default: $(patsubst %,$(HTMLDIR)/%,$(FILES)) $(patsubst %,$(MDDIR)/source/%,$(MDFILES))

gui/icons/Makefile

@@ -7,6 +7,7 @@ SRCICONS := $(wildcard grass/*.png)
 SRCFLAGS := $(wildcard flags/*.png)
 ETCICONS := $(patsubst %,$(GUIDIR)/icons/%,$(SRCICONS) $(SRCICO) $(SRCFLAGS))
 HTMLICONS := $(patsubst grass/%,$(HTMLDIR)/icons/%,$(SRCICONS))
+MDDIRICONS := $(patsubst grass/%,$(MDDIR)/source/icons/%,$(SRCICONS))
 
 DSTDIRS = \
 	$(GUIDIR) \
@@ -17,7 +18,10 @@ DSTDIRS = \
 HTMLDIRS = \
 	$(HTMLDIR)/icons
 
-default: $(ETCICONS) $(HTMLICONS)
+MDDIRS = \
+	$(MDDIR)/source/icons
+
+default: $(ETCICONS) $(HTMLICONS) $(MDDIRICONS)
 	$(MKDIR) $(ARCH_DISTDIR)/share/applications
 	$(INSTALL_DATA) grass.desktop $(ARCH_DISTDIR)/share/applications
 	$(MKDIR) $(ARCH_DISTDIR)/share/icons/hicolor/8x8/apps
@@ -66,5 +70,8 @@ $(GUIDIR)/icons/%: % | $(DSTDIRS)
 $(HTMLDIR)/icons/%: grass/% | $(HTMLDIRS)
 	$(INSTALL_DATA) $< $@
 
-$(DSTDIRS) $(HTMLDIRS): %:
+$(MDDIR)/source/icons/%: grass/% | $(MDDIRS)
+	$(INSTALL_DATA) $< $@
+
+$(DSTDIRS) $(HTMLDIRS) $(MDDIRS): %:
 	$(MKDIR) $@

gui/wxpython/docs/Makefile

@@ -3,5 +3,6 @@ MODULE_TOPDIR = ../../..
 include $(MODULE_TOPDIR)/include/Make/Other.make
 
 FILES := $(wildcard *.html)
+MDFILES := $(wildcard *.md)
 
-default: $(patsubst %,$(HTMLDIR)/%,$(FILES))
+default: $(patsubst %,$(HTMLDIR)/%,$(FILES)) $(patsubst %,$(MDDIR)/source/%,$(MDFILES))

gui/wxpython/gmodeler/Makefile

@@ -6,3 +6,4 @@ default: guiscript sample_model
 
 sample_model: g_gui_gmodeler_zipcodes_avg_elevation.gxm
 	$(INSTALL_DATA) $< $(HTMLDIR)
+	$(INSTALL_DATA) $< $(MDDIR)/source

include/Make/DB.make

@@ -10,7 +10,7 @@ include $(MODULE_TOPDIR)/include/Make/Compile.make
 
 dbmi: $(DBDRIVERDIR)/$(PGM)$(EXE) db_html
 
-db_html: $(HTMLDIR)/grass-$(PGM).html $(MANDIR)/grass-$(PGM).$(MANSECT) # $(MDDIR)/source/grass-$(PGM).md
+db_html: $(HTMLDIR)/grass-$(PGM).html $(MANDIR)/grass-$(PGM).$(MANSECT) $(MDDIR)/source/grass-$(PGM).md
 
 $(DBDRIVERDIR)/$(PGM)$(EXE): $(ARCH_OBJS) $(DEPENDENCIES)
 	$(call linker)

include/Make/GuiScript.make

@@ -27,8 +27,8 @@ ifndef CROSS_COMPILING
 	$(MAKE) $(CMDHTML)
 	-rm -f g.gui.*.tmp.html
 	$(MAKE) $(GUIHTML)
-#	$(MAKE) $(CMDMAN)
-#	$(MAKE) $(GUIMAN)
+	$(MAKE) $(CMDMAN)
+	$(MAKE) $(GUIMAN)
 endif
 
 $(HTMLDIR)/g.gui.%.html: g.gui.%.html g.gui.%.tmp.html | $(HTMLDIR)

include/Make/Html.make

@@ -26,7 +26,7 @@ html:
 
 else
 
-html: $(HTMLDIR)/$(PGM).html $(MANDIR)/$(PGM).$(MANSECT) # $(MDDIR)/source/$(PGM).md
+html: $(HTMLDIR)/$(PGM).html $(MANDIR)/$(PGM).$(MANSECT) $(MDDIR)/source/$(PGM).md
 
 endif
 

include/Make/Multi.make

@@ -8,6 +8,7 @@ include $(MODULE_TOPDIR)/include/Make/Compile.make
 
 PROGFILES = $(patsubst %,$(BIN)/%$(EXE),$(PROGRAMS))
 HTMLFILES = $(patsubst %,$(HTMLDIR)/%.html,$(PROGRAMS))
+MDFILES = $(patsubst %,$(MDDIR)/source/%.md,$(PROGRAMS))
 MANFILES  = $(patsubst %,$(MANDIR)/%.$(MANSECT),$(PROGRAMS))
 
 multi: progs htmlmulti
@@ -17,7 +18,7 @@ progs: $(PROGFILES)
 ifdef CROSS_COMPILING
 htmlmulti:
 else
-htmlmulti: $(HTMLFILES) $(MANFILES)
+htmlmulti: $(HTMLFILES) $(MANFILES) $(MDFILES)
 endif
 
 $(BIN)/%$(EXE): $(DEPENDENCIES)
@@ -28,7 +29,10 @@ $(BIN)/$(1)$(EXE): $$(patsubst %.o,$(OBJDIR)/%.o,$$($$(subst .,_,$(1)_OBJS)))
 $(HTMLDIR)/$(1).html: $(1).html $(1).tmp.html $(BIN)/$(1)$(EXE)
 $(1).tmp.html: $(BIN)/$(1)$(EXE)
 	$$(call htmldesc,$$<,$$@)
-.INTERMEDIATE: $(1).tmp.html
+$(MDDIR)/source/$(1).md: $(1).md $(1).tmp.md $(BIN)/$(1)$(EXE)
+$(1).tmp.md: $(BIN)/$(1)$(EXE)
+	$$(call mddesc,$$<,$$@)
+.INTERMEDIATE: $(1).tmp.html $(1).tmp.md
 endef
 
 $(foreach prog,$(PROGRAMS),$(eval $(call objs_rule,$(prog))))

man/Makefile

@@ -38,11 +38,11 @@ IDXSRC = full_index index topics keywords graphical_index manual_gallery class_g
 
 INDICES := $(patsubst %,$(HTMLDIR)/%.html,$(IDXSRC))
 
-IDXSRC_MD = full_index index topics keywords graphical_index manual_gallery parser_standard_options $(IDXCATS)
+IDXSRC_MD = full_index index topics keywords graphical_index manual_gallery class_graphical parser_standard_options $(IDXCATS)
 INDICES_MD := $(patsubst %,$(MDDIR)/source/%.md,$(IDXSRC_MD))
 
 ALL_HTML := $(wildcard $(HTMLDIR)/*.*.html)
-ALL_MD := $(wildcard $(MDDIR)/source/*.*.html)
+ALL_MD := $(wildcard $(MDDIR)/source/*.*.md)
 
 ifneq (@(type sphinx-build2 > /dev/null),)
 SPHINXBUILD   = sphinx-build2
@@ -125,7 +125,13 @@ $(MDDIR)/source/topics.md: $(ALL_MD)
 define build_class_graphical
 GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \
 	VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \
-	$(PYTHON) ./build_class_graphical.py $(HTMLDIR)
+	$(PYTHON) ./build_class_graphical.py html $(HTMLDIR)
+endef
+
+define build_class_graphical_md
+GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \
+	VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \
+	$(PYTHON) ./build_class_graphical.py md $(MDDIR)/source
 endef
 
 $(HTMLDIR)/topics.html: $(ALL_HTML) build_topics.py
@@ -187,6 +193,11 @@ $(HTMLDIR)/class_graphical.html: $(ALL_HTML)
 	$(call build_class_graphical)
 	touch $@
 
+# TODO: this should be done in the same way as category_rule
+$(MDDIR)/source/class_graphical.md: $(ALL_MD)
+	$(call build_class_graphical_md)
+	touch $@
+
 define category_rule
 $$(HTMLDIR)/$(2).html: $$(wildcard $$(HTMLDIR)/$(1).*.html) build_class.py build_html.py
 	$$(call build,class,$(1) $(2))

man/build_class_graphical.py

@@ -34,9 +34,10 @@
     man_dir,
 )
 
+import build_md
 
-header_graphical_index_tmpl = """\
-<link rel="stylesheet" href="grassdocs.css" type="text/css">
+
+graphical_index_style = """\
 <style>
 .img-list {
     margin: 0;
@@ -83,6 +84,11 @@
     font-style: italic;
 }
 </style>
+"""
+
+header_graphical_index_tmpl = f"""\
+<link rel="stylesheet" href="grassdocs.css" type="text/css">
+{graphical_index_style}
 </head>
 <body style="width: 99%">
 <div id="container">
@@ -190,13 +196,80 @@ def generate_page_for_category(
     replace_file(filename)
 
 
+def generate_page_for_category_md(
+    short_family, module_family, imgs, year, skip_no_image=False
+):
+    filename = module_family + "_graphical.md"
+    with open(filename + ".tmp", "w") as output:
+        output.write(graphical_index_style)
+
+        if module_family.lower() not in {"general", "postscript"}:
+            if module_family == "raster3d":
+                # covert keyword to nice form
+                module_family = "3D raster"
+            output.write(
+                modclass_intro_tmpl.substitute(
+                    modclass=module_family, modclass_lower=module_family.lower()
+                )
+            )
+        if module_family == "wxGUI":
+            output.write("# wxGUI components\n")
+        elif module_family == "guimodules":
+            output.write("# g.gui.* modules\n")
+        else:
+            output.write("# {0} tools\n".format(to_title(module_family)))
+        output.write('<ul class="img-list">')
+
+        # for all modules:
+        for cmd in get_files(
+            build_md.man_dir, short_family, ignore_gui=False, extension="md"
+        ):
+            basename = os.path.splitext(cmd)[0]
+            desc = check_for_desc_override(basename)
+            if desc is None:
+                desc = build_md.get_desc(cmd)
+            img = get_module_image(basename, imgs)
+            img_class = "linkimg"
+            if skip_no_image and not img:
+                continue
+            if not img:
+                img = "grass_logo.png"
+                img_class = "default-img"
+            if basename.startswith("wxGUI"):
+                basename = basename.replace(".", " ")
+            output.write(
+                "<li>"
+                '<a href="{html}.html">'
+                '<img class="{img_class}" src="{img}">'
+                "</a>"
+                '<a href="{html}.html">'
+                '<span class="name">{name}</span> '
+                '<span class="desc">{desc}</span>'
+                "</a>"
+                "</li>".format(
+                    html=cmd.removesuffix(".md"),
+                    img=img,
+                    name=basename,
+                    desc=desc,
+                    img_class=img_class,
+                )
+            )
+
+        output.write("</ul>")
+
+        write_footer(output, "index.html", year, template="md")
+
+    replace_file(filename)
+
+
 # TODO: dependencies in makefile for this have to be fixed
 # TODO: there is a potential overlap with other scripts (-> refactoring)
 
 
 def main():
     year = default_year
-    html_dir = sys.argv[1]
+    output_format = sys.argv[1]
+    html_dir = sys.argv[2]
     os.chdir(html_dir)
 
     img_extensions = ["png", "jpg", "gif"]
@@ -227,16 +300,24 @@ def main():
 
     # partial compatibility with build_class.py
     # first arg is dist html dir but the 3 other are like first 3 there
-    if len(sys.argv) > 2:
-        short_family = sys.argv[2]
-        module_family = sys.argv[3]
-        if len(sys.argv) > 4:
-            year = sys.argv[4]
+    if len(sys.argv) > 3:
+        short_family = sys.argv[3]
+        module_family = sys.argv[4]
+        if len(sys.argv) > 5:
+            year = sys.argv[5]
 
     for short_family, module_family in families:
-        generate_page_for_category(
-            short_family, module_family, imgs, year=year, skip_no_image=False
-        )
+        if output_format == "html":
+            generate_page_for_category(
+                short_family, module_family, imgs, year=year, skip_no_image=False
+            )
+        elif output_format == "md":
+            generate_page_for_category_md(
+                short_family, module_family, imgs, year=year, skip_no_image=False
+            )
+        else:
+            msg = f"Unknown format: {output_format}"
+            raise ValueError(msg)
 
 
 if __name__ == "__main__":

man/build_graphical_index.py

@@ -87,8 +87,8 @@ def main(ext):
                 )
             else:
                 output.write(
-                    "- [![{name}]({img})]({link})".format(
-                        link=html_file, img=image, name=label
+                    "- [![{name}]({img})]({link}.md)".format(
+                        link=html_file.removesuffix(".html"), img=image, name=label
                     )
                 )
 

man/build_topics.py

@@ -61,7 +61,12 @@ def build_topics(ext):
         try:
             desc = lines[index_desc].split("-", 1)[1].strip()
         except Exception:
-            desc.strip()
+            desc = desc.strip()
+
+        # Line ending can appear here on Windows.
+        key = key.strip()
+        if not key:
+            continue
 
         if key not in keywords.keys():
             keywords[key] = {}
@@ -102,7 +107,7 @@ def build_topics(ext):
                     topicsfile.writelines(
                         [
                             moduletopics_tmpl.substitute(
-                                key=key, name=key.replace("_", " ")
+                                key=key.replace(" ", "_"), name=key.replace("_", " ")
                             )
                         ]
                     )
@@ -125,9 +130,10 @@ def build_topics(ext):
                     # expecting markdown
                     keyfile.write(
                         "*See also the corresponding keyword"
-                        " [{key}](keywords.md#{key})"
+                        " [{name}](keywords.md#{key})"
                         " for additional references.*\n".format(
-                            key=key.replace(" ", "-").replace("_", "-").lower()
+                            key=key.replace(" ", "-").replace("_", "-").lower(),
+                            name=key.replace("_", " "),
                         )
                     )
 

raster/Makefile

@@ -44,7 +44,6 @@ SUBDIRS = \
 	r.latlong \
 	r.lake \
 	r.li \
-	r.mapcalc \
 	r.mask.status \
 	r.mfilter \
 	r.mode \