From 5bd41f4c9a83b48bb51fbfd926a0062c0a8aa39a Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 4 Feb 2025 17:07:59 -0500 Subject: [PATCH 01/23] doc: Enable Markdown doc compilation The initial commit simply uncoments the Markdown compilation lines from #3849. This requires the actual Markdown files to be present, so this breaks the CI. This will stay like this for now. The PR is mean to be tested together with the Markdown files generated by script in #4620 (switch to this branch after generating the files). Subsequent commits will add fixes of the compilation (missing tools, files, etc.). --- include/Make/DB.make | 2 +- include/Make/GuiScript.make | 4 ++-- include/Make/Html.make | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/Make/DB.make b/include/Make/DB.make index 797da921ee5..dba3970f7e4 100644 --- a/include/Make/DB.make +++ b/include/Make/DB.make @@ -10,7 +10,7 @@ include $(MODULE_TOPDIR)/include/Make/Compile.make dbmi: $(DBDRIVERDIR)/$(PGM)$(EXE) db_html -db_html: $(HTMLDIR)/grass-$(PGM).html $(MANDIR)/grass-$(PGM).$(MANSECT) # $(MDDIR)/source/grass-$(PGM).md +db_html: $(HTMLDIR)/grass-$(PGM).html $(MANDIR)/grass-$(PGM).$(MANSECT) $(MDDIR)/source/grass-$(PGM).md $(DBDRIVERDIR)/$(PGM)$(EXE): $(ARCH_OBJS) $(DEPENDENCIES) $(call linker) diff --git a/include/Make/GuiScript.make b/include/Make/GuiScript.make index 7451e2b693d..bbe6df85cb7 100644 --- a/include/Make/GuiScript.make +++ b/include/Make/GuiScript.make @@ -27,8 +27,8 @@ ifndef CROSS_COMPILING $(MAKE) $(CMDHTML) -rm -f g.gui.*.tmp.html $(MAKE) $(GUIHTML) -# $(MAKE) $(CMDMAN) -# $(MAKE) $(GUIMAN) + $(MAKE) $(CMDMAN) + $(MAKE) $(GUIMAN) endif $(HTMLDIR)/g.gui.%.html: g.gui.%.html g.gui.%.tmp.html | $(HTMLDIR) diff --git a/include/Make/Html.make b/include/Make/Html.make index 335c841e149..42d5284ecd9 100644 --- a/include/Make/Html.make +++ b/include/Make/Html.make @@ -26,7 +26,7 @@ html: else -html: $(HTMLDIR)/$(PGM).html $(MANDIR)/$(PGM).$(MANSECT) # $(MDDIR)/source/$(PGM).md +html: $(HTMLDIR)/$(PGM).html $(MANDIR)/$(PGM).$(MANSECT) $(MDDIR)/source/$(PGM).md endif From 9217e6e89cb4853f7040ea67e13d8c319ca5045f Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 4 Feb 2025 17:11:53 -0500 Subject: [PATCH 02/23] Add compilation of multi-program directories like r.colors, r.mapcalc, and r3.flow --- include/Make/Multi.make | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/Make/Multi.make b/include/Make/Multi.make index bca014bceb4..bca25733ce2 100644 --- a/include/Make/Multi.make +++ b/include/Make/Multi.make @@ -8,6 +8,7 @@ include $(MODULE_TOPDIR)/include/Make/Compile.make PROGFILES = $(patsubst %,$(BIN)/%$(EXE),$(PROGRAMS)) HTMLFILES = $(patsubst %,$(HTMLDIR)/%.html,$(PROGRAMS)) +MDFILES = $(patsubst %,$(MDDIR)/source/%.md,$(PROGRAMS)) MANFILES = $(patsubst %,$(MANDIR)/%.$(MANSECT),$(PROGRAMS)) multi: progs htmlmulti @@ -17,7 +18,7 @@ progs: $(PROGFILES) ifdef CROSS_COMPILING htmlmulti: else -htmlmulti: $(HTMLFILES) $(MANFILES) +htmlmulti: $(HTMLFILES) $(MANFILES) $(MDFILES) endif $(BIN)/%$(EXE): $(DEPENDENCIES) @@ -28,7 +29,10 @@ $(BIN)/$(1)$(EXE): $$(patsubst %.o,$(OBJDIR)/%.o,$$($$(subst .,_,$(1)_OBJS))) $(HTMLDIR)/$(1).html: $(1).html $(1).tmp.html $(BIN)/$(1)$(EXE) $(1).tmp.html: $(BIN)/$(1)$(EXE) $$(call htmldesc,$$<,$$@) -.INTERMEDIATE: $(1).tmp.html +$(MDDIR)/source/$(1).md: $(1).md $(1).tmp.md $(BIN)/$(1)$(EXE) +$(1).tmp.md: $(BIN)/$(1)$(EXE) + $$(call mddesc,$$<,$$@) +.INTERMEDIATE: $(1).tmp.html $(1).tmp.md endef $(foreach prog,$(PROGRAMS),$(eval $(call objs_rule,$(prog)))) From 8c022a7d4941d4e5eef9a34f57944f15a26f3d01 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 4 Feb 2025 17:15:45 -0500 Subject: [PATCH 03/23] Generate colortable thumbnails. This makes the script little more general and assumes that HTML will be eventually removed (the thumbnails are generated twice now, once for HTML and once of Markdown). --- raster/r.colors/Makefile | 8 ++++++-- utils/thumbnails.py | 5 +++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/raster/r.colors/Makefile b/raster/r.colors/Makefile index de53e51244b..7d691dbbc28 100644 --- a/raster/r.colors/Makefile +++ b/raster/r.colors/Makefile @@ -12,14 +12,18 @@ include $(MODULE_TOPDIR)/include/Make/Multi.make default: multi thumbnails -thumbnails: $(BIN)/r.mapcalc$(EXE) $(BIN)/r.colors$(EXE) $(HTMLDIR)/colortables +thumbnails: $(BIN)/r.mapcalc$(EXE) $(BIN)/r.colors$(EXE) $(HTMLDIR)/colortables $(MDDIR)/source/colortables ifndef CROSS_COMPILING - -$(call run_grass, $(GRASS_HOME)/utils/thumbnails.py) + $(call run_grass, $(GRASS_HOME)/utils/thumbnails.py $(MDDIR)/source/colortables) + $(call run_grass, $(GRASS_HOME)/utils/thumbnails.py $(HTMLDIR)/colortables) endif $(HTMLDIR)/colortables: $(HTMLDIR) $(MKDIR) $@ +$(MDDIR)/source/colortables: $(MDDIR) + $(MKDIR) $@ + $(BIN)/r.mapcalc$(EXE): $(MAKE) -C ../r.mapcalc diff --git a/utils/thumbnails.py b/utils/thumbnails.py index 2eab7319f52..d3be0587dbd 100755 --- a/utils/thumbnails.py +++ b/utils/thumbnails.py @@ -15,6 +15,7 @@ import os import atexit +import sys from pathlib import Path import grass.script as gs @@ -106,7 +107,7 @@ def make_gradient(path): def make_image(output_dir, table, grad, height, width): - outfile = os.path.join(output_dir, "colortables", "%s.png" % table) + outfile = os.path.join(output_dir, "%s.png" % table) os.environ["GRASS_RENDER_FILE"] = outfile gs.run_command("r.colors", map=grad, color=table, quiet=True) @@ -158,7 +159,7 @@ def main(): os.environ["GRASS_OVERWRITE"] = "1" color_dir = os.path.join(os.environ["GISBASE"], "etc", "colors") - output_dir = os.path.join(os.environ["GISBASE"], "docs", "html") + output_dir = sys.argv[1] if not os.path.exists(output_dir): os.makedirs(output_dir) From b68e7f008da7565d64761c78adb276d327616a4d Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 4 Feb 2025 20:41:07 -0500 Subject: [PATCH 04/23] Copy barscale and northarrow files for Markdown doc source --- display/d.barscale/Makefile | 9 ++++++++- display/d.northarrow/Makefile | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/display/d.barscale/Makefile b/display/d.barscale/Makefile index d1e59dddc2e..8cef85c3194 100644 --- a/display/d.barscale/Makefile +++ b/display/d.barscale/Makefile @@ -10,11 +10,18 @@ include $(MODULE_TOPDIR)/include/Make/Module.make # thumbnail previews IMGSRC := $(wildcard thumbnails/*.png) IMGDST := $(patsubst thumbnails/%,$(HTMLDIR)/barscales/%,$(IMGSRC)) +MDIMGDST := $(patsubst thumbnails/%,$(MDDIR)/source/barscales/%,$(IMGSRC)) -default: cmd $(IMGDST) +default: cmd $(IMGDST) $(MDIMGDST) $(HTMLDIR)/barscales/%.png: thumbnails/%.png | $(HTMLDIR)/barscales $(INSTALL_DATA) $< $@ +$(MDDIR)/source/barscales/%.png: thumbnails/%.png | $(MDDIR)/source/barscales + $(INSTALL_DATA) $< $@ + $(HTMLDIR)/barscales: $(HTMLDIR) $(MKDIR) $@ + +$(MDDIR)/source/barscales: $(MDDIR) + $(MKDIR) $@ \ No newline at end of file diff --git a/display/d.northarrow/Makefile b/display/d.northarrow/Makefile index 493446a3289..bf11eda33e7 100644 --- a/display/d.northarrow/Makefile +++ b/display/d.northarrow/Makefile @@ -10,11 +10,18 @@ include $(MODULE_TOPDIR)/include/Make/Module.make # thumbnail previews IMGSRC := $(wildcard thumbnails/*.png) IMGDST := $(patsubst thumbnails/%,$(HTMLDIR)/northarrows/%,$(IMGSRC)) +MDIMGDST := $(patsubst thumbnails/%,$(MDDIR)/source/northarrows/%,$(IMGSRC)) -default: cmd $(IMGDST) +default: cmd $(IMGDST) $(MDIMGDST) $(HTMLDIR)/northarrows/%.png: thumbnails/%.png | $(HTMLDIR)/northarrows $(INSTALL_DATA) $< $@ +$(MDDIR)/source/northarrows/%.png: thumbnails/%.png | $(MDDIR)/source/northarrows + $(INSTALL_DATA) $< $@ + $(HTMLDIR)/northarrows: $(HTMLDIR) $(MKDIR) $@ + +$(MDDIR)/source/northarrows: $(MDDIR) + $(MKDIR) $@ From 0dd9023c67eeaa4fcf01713e3e97d0e1d44d3bd8 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 4 Feb 2025 21:55:11 -0500 Subject: [PATCH 05/23] Presence of spaces vs underscores in key may differ between HTML and Markdown, so always do replacements. --- man/build_topics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/build_topics.py b/man/build_topics.py index 2ef3e8c57df..14962fdca4a 100644 --- a/man/build_topics.py +++ b/man/build_topics.py @@ -102,7 +102,7 @@ def build_topics(ext): topicsfile.writelines( [ moduletopics_tmpl.substitute( - key=key, name=key.replace("_", " ") + key=key.replace(" ", "_"), name=key.replace("_", " ") ) ] ) From c39921ece59b761a112a6d42bf37b29778f579f6 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Tue, 4 Feb 2025 21:57:39 -0500 Subject: [PATCH 06/23] Do not add dash to visible text for keyword. Additionally, remove underscore just like in the HTML version. --- man/build_topics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/man/build_topics.py b/man/build_topics.py index 14962fdca4a..1e41e6ba2f1 100644 --- a/man/build_topics.py +++ b/man/build_topics.py @@ -125,9 +125,10 @@ def build_topics(ext): # expecting markdown keyfile.write( "*See also the corresponding keyword" - " [{key}](keywords.md#{key})" + " [{name}](keywords.md#{key})" " for additional references.*\n".format( - key=key.replace(" ", "-").replace("_", "-").lower() + key=key.replace(" ", "-").replace("_", "-").lower(), + name=key.replace("_", " "), ) ) From 0bd1819114632c3d16a931c58453308c95e24d44 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Wed, 5 Feb 2025 00:12:41 -0500 Subject: [PATCH 07/23] Add projectionintro and grass_database pages from doc dir --- doc/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index 042789d064a..8a5b6e3de1c 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -3,5 +3,8 @@ MODULE_TOPDIR = .. include $(MODULE_TOPDIR)/include/Make/Other.make FILES := $(wildcard *.html) +# So far, we disntinguished user and contributor documentation here by +# extension. This is no longer possible with Markdown. +MDFILES := grass_database.md projectionintro.md -default: $(patsubst %,$(HTMLDIR)/%,$(FILES)) +default: $(patsubst %,$(HTMLDIR)/%,$(FILES)) $(patsubst %,$(MDDIR)/source/%,$(MDFILES)) From 3fac65c90995838c7a1b041247c8b3daea5f6a85 Mon Sep 17 00:00:00 2001 From: Markus Neteler Date: Wed, 5 Feb 2025 12:29:38 +0100 Subject: [PATCH 08/23] d.barscale/Makefile: add EOL --- display/d.barscale/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/display/d.barscale/Makefile b/display/d.barscale/Makefile index 8cef85c3194..7637e8811e0 100644 --- a/display/d.barscale/Makefile +++ b/display/d.barscale/Makefile @@ -24,4 +24,4 @@ $(HTMLDIR)/barscales: $(HTMLDIR) $(MKDIR) $@ $(MDDIR)/source/barscales: $(MDDIR) - $(MKDIR) $@ \ No newline at end of file + $(MKDIR) $@ From aa37a6eec543b83acb50ea81811578f9d05222ae Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Wed, 5 Feb 2025 11:49:24 -0500 Subject: [PATCH 09/23] fix wxGUI docs --- gui/wxpython/docs/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gui/wxpython/docs/Makefile b/gui/wxpython/docs/Makefile index c19abf3fd0e..3561969a0ae 100644 --- a/gui/wxpython/docs/Makefile +++ b/gui/wxpython/docs/Makefile @@ -3,5 +3,6 @@ MODULE_TOPDIR = ../../.. include $(MODULE_TOPDIR)/include/Make/Other.make FILES := $(wildcard *.html) +MDFILES := $(wildcard *.md) -default: $(patsubst %,$(HTMLDIR)/%,$(FILES)) +default: $(patsubst %,$(HTMLDIR)/%,$(FILES)) $(patsubst %,$(MDDIR)/source/%,$(MDFILES))q From c3344a5ab0214e3e9521bf534ec4585cf330e257 Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Wed, 5 Feb 2025 11:56:04 -0500 Subject: [PATCH 10/23] fix typo --- gui/wxpython/docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/wxpython/docs/Makefile b/gui/wxpython/docs/Makefile index 3561969a0ae..e83f3340e14 100644 --- a/gui/wxpython/docs/Makefile +++ b/gui/wxpython/docs/Makefile @@ -5,4 +5,4 @@ include $(MODULE_TOPDIR)/include/Make/Other.make FILES := $(wildcard *.html) MDFILES := $(wildcard *.md) -default: $(patsubst %,$(HTMLDIR)/%,$(FILES)) $(patsubst %,$(MDDIR)/source/%,$(MDFILES))q +default: $(patsubst %,$(HTMLDIR)/%,$(FILES)) $(patsubst %,$(MDDIR)/source/%,$(MDFILES)) From ea1ded618f53376312c529879088f2c602983cee Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Wed, 5 Feb 2025 12:16:17 -0500 Subject: [PATCH 11/23] enable icons --- gui/icons/Makefile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/gui/icons/Makefile b/gui/icons/Makefile index 20a27fb7279..501a378795b 100644 --- a/gui/icons/Makefile +++ b/gui/icons/Makefile @@ -7,6 +7,7 @@ SRCICONS := $(wildcard grass/*.png) SRCFLAGS := $(wildcard flags/*.png) ETCICONS := $(patsubst %,$(GUIDIR)/icons/%,$(SRCICONS) $(SRCICO) $(SRCFLAGS)) HTMLICONS := $(patsubst grass/%,$(HTMLDIR)/icons/%,$(SRCICONS)) +MDDIRICONS := $(patsubst grass/%,$(MDDIR)/source/icons/%,$(SRCICONS)) DSTDIRS = \ $(GUIDIR) \ @@ -17,7 +18,10 @@ DSTDIRS = \ HTMLDIRS = \ $(HTMLDIR)/icons -default: $(ETCICONS) $(HTMLICONS) +MDDIRS = \ + $(MDDIR)/source/icons + +default: $(ETCICONS) $(HTMLICONS) $(MDDIRICONS) $(MKDIR) $(ARCH_DISTDIR)/share/applications $(INSTALL_DATA) grass.desktop $(ARCH_DISTDIR)/share/applications $(MKDIR) $(ARCH_DISTDIR)/share/icons/hicolor/8x8/apps @@ -66,5 +70,8 @@ $(GUIDIR)/icons/%: % | $(DSTDIRS) $(HTMLDIR)/icons/%: grass/% | $(HTMLDIRS) $(INSTALL_DATA) $< $@ -$(DSTDIRS) $(HTMLDIRS): %: +$(MDDIR)/source/icons/%: grass/% | $(MDDIRS) + $(INSTALL_DATA) $< $@ + +$(DSTDIRS) $(HTMLDIRS) $(MDDIRS): %: $(MKDIR) $@ From 179842f6ee1317eaa7acc260f7eee13aa9dff650 Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Wed, 5 Feb 2025 13:52:24 -0500 Subject: [PATCH 12/23] fix modeler file --- gui/wxpython/gmodeler/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/gui/wxpython/gmodeler/Makefile b/gui/wxpython/gmodeler/Makefile index 38f7382ddcb..7deff8050b6 100644 --- a/gui/wxpython/gmodeler/Makefile +++ b/gui/wxpython/gmodeler/Makefile @@ -6,3 +6,4 @@ default: guiscript sample_model sample_model: g_gui_gmodeler_zipcodes_avg_elevation.gxm $(INSTALL_DATA) $< $(HTMLDIR) + $(INSTALL_DATA) $< $(MDDIR)/source From d3a7616e99595bbb540edaf2db84c3fc154ab113 Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Wed, 5 Feb 2025 21:48:08 -0500 Subject: [PATCH 13/23] fix link to grass_logo.txt --- display/d.graph/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/display/d.graph/Makefile b/display/d.graph/Makefile index eb2cd3c1a6c..84c3cc65bfd 100644 --- a/display/d.graph/Makefile +++ b/display/d.graph/Makefile @@ -7,7 +7,10 @@ DEPENDENCIES = $(DISPLAYDEP) $(SYMBDEP) $(GISDEP) include $(MODULE_TOPDIR)/include/Make/Module.make -default: cmd $(HTMLDIR)/grass_logo.txt +default: cmd $(HTMLDIR)/grass_logo.txt $(MDDIR)/source/grass_logo.txt $(HTMLDIR)/%.txt: %.txt $(INSTALL_DATA) $< $@ + +$(MDDIR)/source/%.txt: %.txt + $(INSTALL_DATA) $< $@ From 607f7cddf58fb6368fe552b7d161c7c32d2495ea Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Wed, 5 Feb 2025 22:46:53 -0500 Subject: [PATCH 14/23] Add basic port of graphical index (removes broken links and provides some content) --- man/Makefile | 17 +++++-- man/build_class_graphical.py | 96 +++++++++++++++++++++++++++++++----- 2 files changed, 99 insertions(+), 14 deletions(-) diff --git a/man/Makefile b/man/Makefile index 4ad90b29a74..47bb8bed13e 100644 --- a/man/Makefile +++ b/man/Makefile @@ -38,11 +38,11 @@ IDXSRC = full_index index topics keywords graphical_index manual_gallery class_g INDICES := $(patsubst %,$(HTMLDIR)/%.html,$(IDXSRC)) -IDXSRC_MD = full_index index topics keywords graphical_index manual_gallery parser_standard_options $(IDXCATS) +IDXSRC_MD = full_index index topics keywords graphical_index manual_gallery class_graphical parser_standard_options $(IDXCATS) INDICES_MD := $(patsubst %,$(MDDIR)/source/%.md,$(IDXSRC_MD)) ALL_HTML := $(wildcard $(HTMLDIR)/*.*.html) -ALL_MD := $(wildcard $(MDDIR)/source/*.*.html) +ALL_MD := $(wildcard $(MDDIR)/source/*.*.md) ifneq (@(type sphinx-build2 > /dev/null),) SPHINXBUILD = sphinx-build2 @@ -125,7 +125,13 @@ $(MDDIR)/source/topics.md: $(ALL_MD) define build_class_graphical GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \ VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \ - $(PYTHON) ./build_class_graphical.py $(HTMLDIR) + $(PYTHON) ./build_class_graphical.py html $(HTMLDIR) +endef + +define build_class_graphical_md +GISBASE="$(RUN_GISBASE)" ARCH="$(ARCH)" ARCH_DISTDIR="$(ARCH_DISTDIR)" \ + VERSION_NUMBER=$(GRASS_VERSION_NUMBER) VERSION_DATE=$(GRASS_VERSION_DATE) \ + $(PYTHON) ./build_class_graphical.py md $(MDDIR)/source endef $(HTMLDIR)/topics.html: $(ALL_HTML) build_topics.py @@ -187,6 +193,11 @@ $(HTMLDIR)/class_graphical.html: $(ALL_HTML) $(call build_class_graphical) touch $@ +# TODO: this should be done in the same way as category_rule +$(MDDIR)/source/class_graphical.md: $(ALL_MD) + $(call build_class_graphical_md) + touch $@ + define category_rule $$(HTMLDIR)/$(2).html: $$(wildcard $$(HTMLDIR)/$(1).*.html) build_class.py build_html.py $$(call build,class,$(1) $(2)) diff --git a/man/build_class_graphical.py b/man/build_class_graphical.py index 0eed97beb46..63d8320cfe6 100644 --- a/man/build_class_graphical.py +++ b/man/build_class_graphical.py @@ -34,9 +34,10 @@ man_dir, ) +import build_md -header_graphical_index_tmpl = """\ - + +graphical_index_style = """\ +""" + +header_graphical_index_tmpl = f"""\ + +{graphical_index_style}
@@ -190,13 +196,74 @@ def generate_page_for_category( replace_file(filename) +def generate_page_for_category_md( + short_family, module_family, imgs, year, skip_no_image=False +): + filename = module_family + "_graphical.md" + with open(filename + ".tmp", "w") as output: + output.write(graphical_index_style) + + if module_family.lower() not in {"general", "postscript"}: + if module_family == "raster3d": + # covert keyword to nice form + module_family = "3D raster" + output.write( + modclass_intro_tmpl.substitute( + modclass=module_family, modclass_lower=module_family.lower() + ) + ) + if module_family == "wxGUI": + output.write("# wxGUI components\n") + elif module_family == "guimodules": + output.write("# g.gui.* modules\n") + else: + output.write("# {0} tools\n".format(to_title(module_family))) + output.write('
    ') + + # for all modules: + for cmd in get_files(build_md.man_dir, short_family, ignore_gui=False, extension="md"): + basename = os.path.splitext(cmd)[0] + desc = check_for_desc_override(basename) + if desc is None: + desc = build_md.get_desc(cmd) + img = get_module_image(basename, imgs) + img_class = "linkimg" + if skip_no_image and not img: + continue + if not img: + img = "grass_logo.png" + img_class = "default-img" + if basename.startswith("wxGUI"): + basename = basename.replace(".", " ") + output.write( + "
  • " + '' + '' + '' + '' + '{name} ' + '{desc}' + "" + "
    • ".format( + html=cmd.removesuffix(".md"), img=img, name=basename, desc=desc, img_class=img_class + ) + ) + + output.write("
") + + write_footer(output, "index.html", year, template="md") + + replace_file(filename) + + # TODO: dependencies in makefile for this have to be fixed # TODO: there is a potential overlap with other scripts (-> refactoring) def main(): year = default_year - html_dir = sys.argv[1] + output_format = sys.argv[1] + html_dir = sys.argv[2] os.chdir(html_dir) img_extensions = ["png", "jpg", "gif"] @@ -227,16 +294,23 @@ def main(): # partial compatibility with build_class.py # first arg is dist html dir but the 3 other are like first 3 there - if len(sys.argv) > 2: - short_family = sys.argv[2] - module_family = sys.argv[3] - if len(sys.argv) > 4: - year = sys.argv[4] + if len(sys.argv) > 3: + short_family = sys.argv[3] + module_family = sys.argv[4] + if len(sys.argv) > 5: + year = sys.argv[5] for short_family, module_family in families: - generate_page_for_category( - short_family, module_family, imgs, year=year, skip_no_image=False - ) + if output_format == "html": + generate_page_for_category( + short_family, module_family, imgs, year=year, skip_no_image=False + ) + elif output_format == "md": + generate_page_for_category_md( + short_family, module_family, imgs, year=year, skip_no_image=False + ) + else: + raise ValueError(f"Unknown format: {output_format}") if __name__ == "__main__": From 303ccd87f33f4921b7fe8e781f506c73857589e5 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Wed, 5 Feb 2025 22:59:59 -0500 Subject: [PATCH 15/23] Fix html links in the original md graphical index --- man/build_graphical_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/man/build_graphical_index.py b/man/build_graphical_index.py index bc7dd5299d9..f29feef786b 100755 --- a/man/build_graphical_index.py +++ b/man/build_graphical_index.py @@ -87,8 +87,8 @@ def main(ext): ) else: output.write( - "- [![{name}]({img})]({link})".format( - link=html_file, img=image, name=label + "- [![{name}]({img})]({link}.md)".format( + link=html_file.removesuffix(".html"), img=image, name=label ) ) From 41cb874cbac824c9d88351cb82ff8bb64d6666b8 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Wed, 5 Feb 2025 23:26:05 -0500 Subject: [PATCH 16/23] Do not build r.mapcalc as a subdir because it is build already as a first item --- raster/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/raster/Makefile b/raster/Makefile index a8f00ee9c7c..a571f47f1e2 100644 --- a/raster/Makefile +++ b/raster/Makefile @@ -44,7 +44,6 @@ SUBDIRS = \ r.latlong \ r.lake \ r.li \ - r.mapcalc \ r.mask.status \ r.mfilter \ r.mode \ From 06930da7b663a108bcd29a73c284032243ebc10a Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Thu, 6 Feb 2025 09:09:44 -0500 Subject: [PATCH 17/23] Fix formatting Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- man/build_class_graphical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/build_class_graphical.py b/man/build_class_graphical.py index 63d8320cfe6..f86ab236146 100644 --- a/man/build_class_graphical.py +++ b/man/build_class_graphical.py @@ -86,7 +86,7 @@ """ -header_graphical_index_tmpl = f"""\ +header_graphical_index_tmpl = f"""\ {graphical_index_style} From 75f6f8db70633e7e46984bbe2cd274a9d03597ac Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Thu, 6 Feb 2025 09:10:23 -0500 Subject: [PATCH 18/23] Fix raise statement usage Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- man/build_class_graphical.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/man/build_class_graphical.py b/man/build_class_graphical.py index f86ab236146..34e6745d674 100644 --- a/man/build_class_graphical.py +++ b/man/build_class_graphical.py @@ -310,7 +310,8 @@ def main(): short_family, module_family, imgs, year=year, skip_no_image=False ) else: - raise ValueError(f"Unknown format: {output_format}") + msg = f"Unknown format: {output_format}" + raise ValueError(msg) if __name__ == "__main__": From e38c57a2164b2a00953b978011413a71a93690fb Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Thu, 6 Feb 2025 23:18:46 -0500 Subject: [PATCH 19/23] Fix Python code formatting --- man/build_class_graphical.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/man/build_class_graphical.py b/man/build_class_graphical.py index 34e6745d674..3cfc1d6ec6d 100644 --- a/man/build_class_graphical.py +++ b/man/build_class_graphical.py @@ -221,7 +221,9 @@ def generate_page_for_category_md( output.write('
    ') # for all modules: - for cmd in get_files(build_md.man_dir, short_family, ignore_gui=False, extension="md"): + for cmd in get_files( + build_md.man_dir, short_family, ignore_gui=False, extension="md" + ): basename = os.path.splitext(cmd)[0] desc = check_for_desc_override(basename) if desc is None: @@ -239,13 +241,17 @@ def generate_page_for_category_md( "
  • " '' '' - '' + "" '' '{name} ' '{desc}' "" "
    • ".format( - html=cmd.removesuffix(".md"), img=img, name=basename, desc=desc, img_class=img_class + html=cmd.removesuffix(".md"), + img=img, + name=basename, + desc=desc, + img_class=img_class ) ) From c68886eb806d55125f7ee72a524be268ec8f819e Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Thu, 6 Feb 2025 23:27:20 -0500 Subject: [PATCH 20/23] Strip keywords from whitespace and skip empty ones --- man/build_topics.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/man/build_topics.py b/man/build_topics.py index 1e41e6ba2f1..555945667db 100644 --- a/man/build_topics.py +++ b/man/build_topics.py @@ -61,7 +61,12 @@ def build_topics(ext): try: desc = lines[index_desc].split("-", 1)[1].strip() except Exception: - desc.strip() + desc = desc.strip() + + # Line ending can appear here. + key = key.strip() + if not key: + continue if key not in keywords.keys(): keywords[key] = {} From ebab229f9d9a074209d717664616eb547be17fe1 Mon Sep 17 00:00:00 2001 From: Vaclav Petras Date: Thu, 6 Feb 2025 23:29:32 -0500 Subject: [PATCH 21/23] More precise comment, fix format --- man/build_class_graphical.py | 2 +- man/build_topics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/man/build_class_graphical.py b/man/build_class_graphical.py index 3cfc1d6ec6d..47ef979d9fe 100644 --- a/man/build_class_graphical.py +++ b/man/build_class_graphical.py @@ -251,7 +251,7 @@ def generate_page_for_category_md( img=img, name=basename, desc=desc, - img_class=img_class + img_class=img_class, ) ) diff --git a/man/build_topics.py b/man/build_topics.py index 555945667db..a9a42ea3108 100644 --- a/man/build_topics.py +++ b/man/build_topics.py @@ -63,7 +63,7 @@ def build_topics(ext): except Exception: desc = desc.strip() - # Line ending can appear here. + # Line ending can appear here on Windows. key = key.strip() if not key: continue From ce9a154ffe6796bc47d2debdc67d64f103a9bf0f Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Fri, 7 Feb 2025 14:28:45 -0500 Subject: [PATCH 22/23] g.extension: create md from html on the fly if no md --- scripts/g.extension/g.extension.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index f694e73c23a..afa3f17a60c 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -1957,6 +1957,23 @@ def download_source_code( return directory, url +def create_md_if_missing(root_dir): + """Recursively searches for HTML files in the specified directory. + If an HTML file does not have a corresponding Markdown (.md) file, + it creates one by copying the HTML file and renaming it. + """ + for dirpath, _, filenames in os.walk(root_dir): + html_files = [f for f in filenames if f.endswith(".html")] + + for html_file in html_files: + md_file = os.path.splitext(html_file)[0] + ".md" + md_path = os.path.join(dirpath, md_file) + + if not os.path.exists(md_path): + html_path = os.path.join(dirpath, html_file) + shutil.copy(html_path, md_path) + + def install_extension_std_platforms(name, source, url, branch): """Install extension on standard platforms""" gisbase = os.getenv("GISBASE") @@ -1976,6 +1993,7 @@ def install_extension_std_platforms(name, source, url, branch): tmpdir=TMPDIR, branch=branch, ) + create_md_if_missing(srcdir) os.chdir(srcdir) pgm_not_found_message = _( From 7e0daf72228280c778d788a0cd3acc7918d64ff7 Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Tue, 11 Feb 2025 10:42:11 -0500 Subject: [PATCH 23/23] attempt to fix race condition in g.extension --- utils/mkdocs.py | 141 +++++++++++++++++++++++++----------------------- 1 file changed, 75 insertions(+), 66 deletions(-) diff --git a/utils/mkdocs.py b/utils/mkdocs.py index 1323fe06229..ecbca5f4a0f 100644 --- a/utils/mkdocs.py +++ b/utils/mkdocs.py @@ -1,14 +1,16 @@ # common functions used by mkmarkdown.py and mkhtml.py -import sys -import os import json -import subprocess +import os import re +import shutil +import subprocess +import sys +import tempfile import urllib.parse as urlparse +from datetime import datetime from http import HTTPStatus from pathlib import Path -from datetime import datetime from urllib import request as urlrequest from urllib.error import HTTPError, URLError @@ -368,68 +370,75 @@ def get_addon_path(base_url, pgm, major_version): :return str|None: pgm path if pgm is addon else None """ addons_base_dir = os.getenv("GRASS_ADDON_BASE") - if addons_base_dir and major_version: - grass_addons_dir = Path(addons_base_dir) / "grass-addons" - if gs: - call = gs.call - popen = gs.Popen - fatal = gs.fatal - else: - call = subprocess.call - popen = subprocess.Popen - fatal = sys.stderr.write - addons_branch = get_version_branch( - major_version=major_version, - addons_git_repo_url=urlparse.urljoin(base_url, "grass-addons/"), - ) - if not Path(addons_base_dir).exists(): - Path(addons_base_dir).mkdir(parents=True, exist_ok=True) - if not grass_addons_dir.exists(): - call( - [ - "git", - "clone", - "-q", - "--no-checkout", - f"--branch={addons_branch}", - "--filter=blob:none", - urlparse.urljoin(base_url, "grass-addons/"), - ], - cwd=addons_base_dir, - ) - addons_file_list = popen( - ["git", "ls-tree", "--name-only", "-r", addons_branch], - cwd=grass_addons_dir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + if not addons_base_dir or not major_version: + return None + grass_addons_dir = Path(addons_base_dir) / "grass-addons" + if gs: + call = gs.call + popen = gs.Popen + fatal = gs.fatal + else: + call = subprocess.call + popen = subprocess.Popen + fatal = sys.stderr.write + addons_branch = get_version_branch( + major_version=major_version, + addons_git_repo_url=urlparse.urljoin(base_url, "grass-addons/"), + ) + if not Path(addons_base_dir).exists(): + Path(addons_base_dir).mkdir(parents=True, exist_ok=True) + if not grass_addons_dir.exists(): + try: + with tempfile.TemporaryDirectory(dir=addons_base_dir) as tmpdir: + tmp_clone_path = Path(tmpdir) / "grass-addons" + call( + [ + "git", + "clone", + "-q", + "--no-checkout", + f"--branch={addons_branch}", + "--filter=blob:none", + urlparse.urljoin(base_url, "grass-addons/"), + str(tmp_clone_path), + ] + ) + shutil.move(tmp_clone_path, grass_addons_dir) + except shutil.Error: + pass # Another process created it first; proceed normally + addons_file_list = popen( + ["git", "ls-tree", "--name-only", "-r", addons_branch], + cwd=grass_addons_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + addons_file_list, stderr = addons_file_list.communicate() + if stderr: + message = ( + "Failed to get addons files list from the" + " Git repository <{repo_path}>.\n{error}" ) - addons_file_list, stderr = addons_file_list.communicate() - if stderr: - message = ( - "Failed to get addons files list from the" - " Git repository <{repo_path}>.\n{error}" - ) - if gs: - fatal( - _( - message, - ).format( - repo_path=grass_addons_dir, - error=gs.decode(stderr), - ) + if gs: + fatal( + _( + message, + ).format( + repo_path=grass_addons_dir, + error=gs.decode(stderr), ) - else: - message += "\n" - fatal( - message.format( - repo_path=grass_addons_dir, - error=stderr.decode(), - ) + ) + else: + message += "\n" + fatal( + message.format( + repo_path=grass_addons_dir, + error=stderr.decode(), ) - addon_paths = re.findall( - rf".*{pgm}*.", - gs.decode(addons_file_list) if gs else addons_file_list.decode(), - ) - for addon_path in addon_paths: - if pgm == Path(addon_path).name: - return addon_path + ) + addon_paths = re.findall( + rf".*{pgm}*.", + gs.decode(addons_file_list) if gs else addons_file_list.decode(), + ) + for addon_path in addon_paths: + if pgm == Path(addon_path).name: + return addon_path