From cc4e522f30434b19635425ee9573c2675ae41608 Mon Sep 17 00:00:00 2001 From: Martin Landa Date: Wed, 31 Jul 2024 18:58:35 +0200 Subject: [PATCH 01/53] patch from https://github.com/jef-n/OSGeo4W/blob/master/src/grass/osgeo4w/patch applied --- .github/workflows/osgeo4w.yml | 5 ++-- include/Make/Grass.make | 2 +- include/Make/Platform.make.in | 2 +- mswindows/osgeo4w/build_osgeo4w.sh | 6 ++-- mswindows/osgeo4w/env.bat.tmpl | 2 ++ mswindows/osgeo4w/package.sh | 47 +++++++++++++----------------- 6 files changed, 31 insertions(+), 33 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 2fd1412d76b..d1500dcee02 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -40,9 +40,10 @@ jobs: mingw-w64-x86_64-fftw mingw-w64-x86_64-lapack mingw-w64-x86_64-pkgconf mingw-w64-x86_64-gcc mingw-w64-x86_64-ccache mingw-w64-x86_64-zlib mingw-w64-x86_64-libiconv mingw-w64-x86_64-bzip2 mingw-w64-x86_64-gettext mingw-w64-x86_64-libsystre - mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-libpng + mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-pcre + - name: Install OSGeo4W run: | $exe = 'osgeo4w-setup.exe' @@ -55,7 +56,7 @@ jobs: pdal-devel,netcdf-devel,cairo-devel,fftw,freetype-devel,gdal-ecw,\ gdal-mrsid,liblas-devel,libxdr,libpq-devel,pdcurses,\ python3-matplotlib,python3-numpy,python3-ply,python3-pywin32,\ - python3-wxpython,regex-devel,zstd-devel" + python3-wxpython,zstd-devel" - name: Set number of cores for compilation run: | diff --git a/include/Make/Grass.make b/include/Make/Grass.make index 137a45d1955..4f26feda214 100644 --- a/include/Make/Grass.make +++ b/include/Make/Grass.make @@ -197,7 +197,7 @@ DSPFDEPS = $(GISLIB) FORMDEPS = $(DBMILIB) $(GISLIB) RASTER3DDEPS = $(RASTERLIB) $(GISLIB) GISDEPS = $(DATETIMELIB) $(ZLIBLIBPATH) $(ZLIB) $(BZIP2LIBPATH) $(BZIP2LIB) $(ZSTDLIBPATH) $(ZSTDLIB) $(INTLLIB) $(REGEXLIBPATH) $(REGEXLIB) $(ICONVLIB) $(PTHREADLIBPATH) $(PTHREADLIB) $(MATHLIB) -GMATHDEPS = $(GISLIB) $(FFTWLIB) $(LAPACKLIB) $(BLASLIB) $(CCMATHLIB) $(OPENMP_CFLAGS) $(OPENMP_LIBPATH) $(OPENMP_LIB) +GMATHDEPS = $(GISLIB) $(FFTWLIB) $(LAPACKLIB) $(BLASLIB) $(CCMATHLIB) $(OPENMP_LIBPATH) $(OPENMP_LIB) GPDEDEPS = $(RASTER3DLIB) $(RASTERLIB) $(GISLIB) $(GMATHLIB) $(OPENMP_LIBPATH) $(OPENMP_LIB) $(MATHLIB) GPROJDEPS = $(GISLIB) $(GDALLIBS) $(PROJLIB) $(MATHLIB) HTMLDRIVERDEPS = $(DRIVERLIB) $(GISLIB) $(MATHLIB) diff --git a/include/Make/Platform.make.in b/include/Make/Platform.make.in index c65ed28c686..10276da6e5a 100644 --- a/include/Make/Platform.make.in +++ b/include/Make/Platform.make.in @@ -234,7 +234,7 @@ USE_PTHREAD = @USE_PTHREAD@ #OpenMP OPENMP_INCPATH = @OPENMP_INCPATH@ OPENMP_LIBPATH = @OPENMP_LIBPATH@ -OPENMP_LIB = @OPENMP_LIB@ +OPENMP_LIB = @OPENMP_CFLAGS@ @OPENMP_LIB@ OPENMP_CFLAGS = @OPENMP_CFLAGS@ USE_OPENMP = @USE_OPENMP@ diff --git a/mswindows/osgeo4w/build_osgeo4w.sh b/mswindows/osgeo4w/build_osgeo4w.sh index 6885bfe02f8..24a46d16288 100755 --- a/mswindows/osgeo4w/build_osgeo4w.sh +++ b/mswindows/osgeo4w/build_osgeo4w.sh @@ -38,7 +38,8 @@ export ARCH=x86_64-w64-mingw32 --with-nls \ --with-readline \ --with-blas \ - --with-lapack-includes=/mingw64/include/lapack \ + --with-lapack \ + --with-lapack-includes=/mingw64/include \ --with-freetype \ --with-freetype-includes=${OSGEO4W_ROOT_MSYS}/include/freetype2 \ --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \ @@ -53,12 +54,11 @@ export ARCH=x86_64-w64-mingw32 --with-sqlite-includes=${OSGEO4W_ROOT_MSYS}/include \ --with-sqlite-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-regex \ - --with-nls \ --with-zstd \ --with-odbc \ --with-cairo \ --with-cairo-includes=${OSGEO4W_ROOT_MSYS}/include \ - --with-cairo-libs=$OSGEO4W_ROOT_MSYS/lib \ + --with-cairo-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-cairo-ldflags="-L${SRC}/mswindows/osgeo4w/lib -lcairo" \ --with-opengl=windows \ --with-bzlib \ diff --git a/mswindows/osgeo4w/env.bat.tmpl b/mswindows/osgeo4w/env.bat.tmpl index 200c86f2687..04065161a78 100644 --- a/mswindows/osgeo4w/env.bat.tmpl +++ b/mswindows/osgeo4w/env.bat.tmpl @@ -8,6 +8,8 @@ REM Uncomment if you want to use Bash instead of Cmd REM Note that msys package must be also installed REM set GRASS_SH=%OSGEO4W_ROOT%\apps\msys\bin\sh.exe +set PYTHONPATH=%OSGEO4W_ROOT%\apps\grass\grass@POSTFIX@\etc\python;%PYTHONPATH% +set GRASS_COMPATIBILITY_TEST=0 set GRASS_PYTHON=%OSGEO4W_ROOT%\bin\python3.exe set GRASS_PROJSHARE=%OSGEO4W_ROOT%\share\proj diff --git a/mswindows/osgeo4w/package.sh b/mswindows/osgeo4w/package.sh index b3113292c99..713396f3cc6 100755 --- a/mswindows/osgeo4w/package.sh +++ b/mswindows/osgeo4w/package.sh @@ -110,29 +110,21 @@ DLLS=" /mingw64/bin/zlib1.dll /mingw64/bin/libbz2-1.dll /mingw64/bin/libiconv-2.dll - /mingw64/bin/libfontconfig-1.dll /mingw64/bin/libgfortran-5.dll - /mingw64/bin/libbrotlidec.dll - /mingw64/bin/libbrotlicommon.dll /mingw64/bin/libintl-8.dll /mingw64/bin/libsystre-0.dll /mingw64/bin/libtre-5.dll /mingw64/bin/libwinpthread-1.dll - /mingw64/bin/libcairo-2.dll - /mingw64/bin/libpixman-1-0.dll /mingw64/bin/libpng16-16.dll - /mingw64/bin/libfreetype-6.dll - /mingw64/bin/libharfbuzz-0.dll - /mingw64/bin/libglib-2.0-0.dll /mingw64/bin/libgomp-1.dll - /mingw64/bin/libgraphite2.dll - /mingw64/bin/libpcre-1.dll + /mingw64/bin/libomp.dll /mingw64/bin/libstdc++-6.dll /mingw64/bin/libgcc_s_seh-1.dll /mingw64/bin/libfftw3-3.dll /mingw64/bin/libblas.dll /mingw64/bin/liblapack.dll /mingw64/bin/libquadmath-0.dll + /mingw64/bin/libreadline8.dll " if ! [ -f mswindows/osgeo4w/configure-stamp ]; then @@ -148,18 +140,18 @@ if ! [ -f mswindows/osgeo4w/configure-stamp ]; then cp -uv $DLLS dist.x86_64-w64-mingw32/bin mkdir -p mswindows/osgeo4w/lib - cp -uv $OSGEO4W_ROOT_MSYS/lib/libpq.lib mswindows/osgeo4w/lib/pq.lib cp -uv $OSGEO4W_ROOT_MSYS/lib/sqlite3_i.lib mswindows/osgeo4w/lib/sqlite3.lib + log configure ./configure \ --host=x86_64-w64-mingw32 \ - --with-libs="$OSGEO4W_ROOT_MSYS/lib" \ - --with-includes=$OSGEO4W_ROOT_MSYS/include \ - --libexecdir=$OSGEO4W_ROOT_MSYS/bin \ - --prefix=$OSGEO4W_ROOT_MSYS/apps/grass \ - --bindir=$OSGEO4W_ROOT_MSYS/bin \ - --includedir=$OSGEO4W_ROOT_MSYS/include \ + --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \ + --with-includes=${OSGEO4W_ROOT_MSYS}/include \ + --libexecdir=${OSGEO4W_ROOT_MSYS}/bin \ + --prefix=${OSGEO4W_ROOT_MSYS}/apps/grass \ + --bindir=${OSGEO4W_ROOT_MSYS}/bin \ + --includedir=${OSGEO4W_ROOT_MSYS}/include \ --with-opengl=windows \ --without-x \ --with-cxx \ @@ -167,13 +159,13 @@ if ! [ -f mswindows/osgeo4w/configure-stamp ]; then --enable-largefile \ --with-fftw \ --with-freetype \ - --with-freetype-includes=/mingw64/include/freetype2 \ - --with-proj-share=$OSGEO4W_ROOT_MSYS/share/proj \ - --with-proj-includes=$OSGEO4W_ROOT_MSYS/include \ - --with-proj-libs=$OSGEO4W_ROOT_MSYS/lib \ + --with-freetype-includes=${OSGEO4W_ROOT_MSYS}/include/freetype2 \ + --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \ + --with-proj-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-proj-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-postgres \ - --with-postgres-includes=$OSGEO4W_ROOT_MSYS/include \ - --with-postgres-libs=$PWD/mswindows/osgeo4w/lib \ + --with-postgres-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-postgres-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-gdal=$PWD/mswindows/osgeo4w/gdal-config \ --with-geos=$PWD/mswindows/osgeo4w/geos-config \ --with-sqlite \ @@ -189,10 +181,11 @@ if ! [ -f mswindows/osgeo4w/configure-stamp ]; then --with-lapack-includes=/mingw64/include \ --with-openmp \ --with-cairo \ - --with-cairo-includes=$OSGEO4W_ROOT_MSYS/include \ - --with-cairo-ldflags="-L$PWD/mswindows/osgeo4w/lib -lcairo -lfontconfig" \ + --with-cairo-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-cairo-ldflags="-L$PWD/mswindows/osgeo4w/lib -lcairo" \ --with-bzlib \ --with-liblas=$PWD/mswindows/osgeo4w/liblas-config \ + --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config \ --without-pdal touch mswindows/osgeo4w/configure-stamp @@ -248,7 +241,9 @@ if [ -n "$PACKAGE_PATCH" ]; then # copy dependencies (TODO: to be reduced) cp -uv $DLLS apps/grass/grass$POSTFIX/bin - cp -uv /mingw64/etc/fonts/fonts.conf apps/grass/grass$POSTFIX/etc + + # copy R batch files + cp -uv $SRC/mswindows/external/rbatch/* apps/grass/grass$POSTFIX/bin # creating grass package /bin/tar -cjf $PDIR/grass$PACKAGE_POSTFIX-$VERSION-$PACKAGE_PATCH.tar.bz2 \ From 42ef64623207cd6b515643e7041a679c2e486b97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20de=20Sousa?= Date: Wed, 31 Jul 2024 19:58:13 +0100 Subject: [PATCH 02/53] v.out.ogr: Add basic unit tests (#3848) Add basic tests for GeoPackage and Shapefile export which use import to test the result (so they test round trip but focus on the export). --- vector/v.out.ogr/testsuite/test_v_out_ogr.py | 115 +++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 vector/v.out.ogr/testsuite/test_v_out_ogr.py diff --git a/vector/v.out.ogr/testsuite/test_v_out_ogr.py b/vector/v.out.ogr/testsuite/test_v_out_ogr.py new file mode 100644 index 00000000000..d269ccff606 --- /dev/null +++ b/vector/v.out.ogr/testsuite/test_v_out_ogr.py @@ -0,0 +1,115 @@ +"""Test of v.out.ogr + +@author Luís Moreira de Sousa +""" + +from pathlib import Path +from grass.gunittest.case import TestCase + + +class TestOgrExport(TestCase): + + # Vector map in NC test dataset + test_map = "boundary_county" + + # Result of import tests + temp_import = "test_ogr_import_map" + + # Column on which to test v.univar + univar_col = "PERIMETER" + + # Output of v.univar + univar_string = """n=926 +nmissing=0 +nnull=0 +min=9.64452 +max=3.70609e+06 +range=3.70608e+06 +sum=1.1223e+08 +mean=121199 +mean_abs=121199 +population_stddev=342855 +population_variance=1.17549e+11 +population_coeff_variation=2.82886 +sample_stddev=343040 +sample_variance=1.17676e+11 +kurtosis=33.681 +skewness=4.86561 +""" + + @classmethod + def setUpClass(cls): + """Use temporary region settings""" + cls.use_temp_region() + + @classmethod + def tearDownClass(cls): + """Remove the temporary region""" + cls.del_temp_region() + + def tearDown(self): + self.runModule( + "g.remove", type="vector", flags="f", pattern=f"{self.temp_import}*" + ) + for p in Path().glob(f"{self.test_map}*"): + p.unlink() + + def test_gpkg_format(self): + """Tests output to GeoPackage format""" + + self.assertModule( + "v.out.ogr", + "Export to GeoPackage Format", + input=self.test_map, + output=f"{self.test_map}.gpkg", + format="GPKG", + ) + + # Import back to verify + self.runModule( + "v.in.ogr", + input=f"{self.test_map}.gpkg", + output=self.temp_import, + ) + + self.runModule("g.region", vector=self.temp_import) + + self.assertVectorFitsUnivar( + map=self.temp_import, + reference=self.univar_string, + column=self.univar_col, + precision=1e-8, + ) + + def test_shp_format(self): + """Tests output to Shapefile format""" + + self.assertModule( + "v.out.ogr", + "Export to Shapefile Format", + input=self.test_map, + output=f"{self.test_map}.shp", + format="ESRI_Shapefile", + ) + + # Import back to verify + self.runModule( + "v.in.ogr", + input=f"{self.test_map}.shp", + output=self.temp_import, + ) + + self.runModule("g.region", vector=self.temp_import) + + self.assertVectorFitsUnivar( + map=self.temp_import, + reference=self.univar_string, + column=self.univar_col, + precision=1e-8, + ) + + +if __name__ == "__main__": + from grass.gunittest.main import test + + test() From 3b2313f62250c0c9fe93f25724333a5a61092488 Mon Sep 17 00:00:00 2001 From: Martin Landa Date: Wed, 31 Jul 2024 21:45:00 +0200 Subject: [PATCH 03/53] attempt to fix osgeo4w CI --- mswindows/osgeo4w/build_osgeo4w.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/mswindows/osgeo4w/build_osgeo4w.sh b/mswindows/osgeo4w/build_osgeo4w.sh index 24a46d16288..caaaa6e8bbd 100755 --- a/mswindows/osgeo4w/build_osgeo4w.sh +++ b/mswindows/osgeo4w/build_osgeo4w.sh @@ -40,6 +40,7 @@ export ARCH=x86_64-w64-mingw32 --with-blas \ --with-lapack \ --with-lapack-includes=/mingw64/include \ + --with-libpng=${SRC}/mswindows/osgeo4w/libpng-config \ --with-freetype \ --with-freetype-includes=${OSGEO4W_ROOT_MSYS}/include/freetype2 \ --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \ From fd270e0d86c11806c24fef614198ec75a31367ca Mon Sep 17 00:00:00 2001 From: Martin Landa Date: Wed, 31 Jul 2024 21:47:36 +0200 Subject: [PATCH 04/53] Revert "v.out.ogr: Add basic unit tests (#3848)" This reverts commit 42ef64623207cd6b515643e7041a679c2e486b97. --- vector/v.out.ogr/testsuite/test_v_out_ogr.py | 115 ------------------- 1 file changed, 115 deletions(-) delete mode 100644 vector/v.out.ogr/testsuite/test_v_out_ogr.py diff --git a/vector/v.out.ogr/testsuite/test_v_out_ogr.py b/vector/v.out.ogr/testsuite/test_v_out_ogr.py deleted file mode 100644 index d269ccff606..00000000000 --- a/vector/v.out.ogr/testsuite/test_v_out_ogr.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Test of v.out.ogr - -@author Luís Moreira de Sousa -""" - -from pathlib import Path -from grass.gunittest.case import TestCase - - -class TestOgrExport(TestCase): - - # Vector map in NC test dataset - test_map = "boundary_county" - - # Result of import tests - temp_import = "test_ogr_import_map" - - # Column on which to test v.univar - univar_col = "PERIMETER" - - # Output of v.univar - univar_string = """n=926 -nmissing=0 -nnull=0 -min=9.64452 -max=3.70609e+06 -range=3.70608e+06 -sum=1.1223e+08 -mean=121199 -mean_abs=121199 -population_stddev=342855 -population_variance=1.17549e+11 -population_coeff_variation=2.82886 -sample_stddev=343040 -sample_variance=1.17676e+11 -kurtosis=33.681 -skewness=4.86561 -""" - - @classmethod - def setUpClass(cls): - """Use temporary region settings""" - cls.use_temp_region() - - @classmethod - def tearDownClass(cls): - """Remove the temporary region""" - cls.del_temp_region() - - def tearDown(self): - self.runModule( - "g.remove", type="vector", flags="f", pattern=f"{self.temp_import}*" - ) - for p in Path().glob(f"{self.test_map}*"): - p.unlink() - - def test_gpkg_format(self): - """Tests output to GeoPackage format""" - - self.assertModule( - "v.out.ogr", - "Export to GeoPackage Format", - input=self.test_map, - output=f"{self.test_map}.gpkg", - format="GPKG", - ) - - # Import back to verify - self.runModule( - "v.in.ogr", - input=f"{self.test_map}.gpkg", - output=self.temp_import, - ) - - self.runModule("g.region", vector=self.temp_import) - - self.assertVectorFitsUnivar( - map=self.temp_import, - reference=self.univar_string, - column=self.univar_col, - precision=1e-8, - ) - - def test_shp_format(self): - """Tests output to Shapefile format""" - - self.assertModule( - "v.out.ogr", - "Export to Shapefile Format", - input=self.test_map, - output=f"{self.test_map}.shp", - format="ESRI_Shapefile", - ) - - # Import back to verify - self.runModule( - "v.in.ogr", - input=f"{self.test_map}.shp", - output=self.temp_import, - ) - - self.runModule("g.region", vector=self.temp_import) - - self.assertVectorFitsUnivar( - map=self.temp_import, - reference=self.univar_string, - column=self.univar_col, - precision=1e-8, - ) - - -if __name__ == "__main__": - from grass.gunittest.main import test - - test() From daba1e6c55b815d609264af52c9a76905f9eecce Mon Sep 17 00:00:00 2001 From: Martin Landa Date: Sun, 4 Aug 2024 07:51:14 +0200 Subject: [PATCH 05/53] re-introduce mingw-w64-x86_64-libpng --- .github/workflows/osgeo4w.yml | 2 +- mswindows/osgeo4w/build_osgeo4w.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index d1500dcee02..a21efef7f57 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -40,7 +40,7 @@ jobs: mingw-w64-x86_64-fftw mingw-w64-x86_64-lapack mingw-w64-x86_64-pkgconf mingw-w64-x86_64-gcc mingw-w64-x86_64-ccache mingw-w64-x86_64-zlib mingw-w64-x86_64-libiconv mingw-w64-x86_64-bzip2 mingw-w64-x86_64-gettext mingw-w64-x86_64-libsystre - mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git + mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-libpng mingw-w64-x86_64-pcre diff --git a/mswindows/osgeo4w/build_osgeo4w.sh b/mswindows/osgeo4w/build_osgeo4w.sh index caaaa6e8bbd..24a46d16288 100755 --- a/mswindows/osgeo4w/build_osgeo4w.sh +++ b/mswindows/osgeo4w/build_osgeo4w.sh @@ -40,7 +40,6 @@ export ARCH=x86_64-w64-mingw32 --with-blas \ --with-lapack \ --with-lapack-includes=/mingw64/include \ - --with-libpng=${SRC}/mswindows/osgeo4w/libpng-config \ --with-freetype \ --with-freetype-includes=${OSGEO4W_ROOT_MSYS}/include/freetype2 \ --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \ From 56dc8efb3bd6b791af6f5423e47725b0ef655098 Mon Sep 17 00:00:00 2001 From: Martin Landa Date: Tue, 6 Aug 2024 08:36:24 +0200 Subject: [PATCH 06/53] sync with osgeo4w patch --- .github/workflows/osgeo4w.yml | 11 +++++------ mswindows/osgeo4w/build_osgeo4w.sh | 1 + mswindows/osgeo4w/libpng-config | 2 +- mswindows/osgeo4w/package.sh | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 5b9fab74725..a5302d50dab 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -40,7 +40,7 @@ jobs: mingw-w64-x86_64-fftw mingw-w64-x86_64-lapack mingw-w64-x86_64-pkgconf mingw-w64-x86_64-gcc mingw-w64-x86_64-ccache mingw-w64-x86_64-zlib mingw-w64-x86_64-libiconv mingw-w64-x86_64-bzip2 mingw-w64-x86_64-gettext mingw-w64-x86_64-libsystre - mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-libpng + mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-pcre @@ -52,11 +52,10 @@ jobs: Start-Process ('.\'+$exe) -ArgumentList '-A -g -k -q \ -s http://download.osgeo.org/osgeo4w/v2/ -P ${{ env.Deps }}' -Wait env: - Deps: "proj-devel,gdal-devel,geos-devel,libtiff-devel,libpng-devel,\ - pdal-devel,netcdf-devel,cairo-devel,fftw,freetype-devel,gdal-ecw,\ - gdal-mrsid,liblas-devel,libxdr,libpq-devel,pdcurses,\ - python3-matplotlib,python3-numpy,python3-ply,python3-pywin32,\ - python3-wxpython,zstd-devel" + Deps: "gdal-devel,proj-devel,geos-devel,netcdf-devel,libjpeg-turbo-devel,\ + libpq-devel,libpng-devel,libtiff-devel,sqlite3-devel,zstd-devel,python3-ply,\ + python3-core,python3-six,python3-pywin32,python3-wxpython,liblas-devel,\ + cairo-devel,freetype-devel" - name: Set number of cores for compilation run: | diff --git a/mswindows/osgeo4w/build_osgeo4w.sh b/mswindows/osgeo4w/build_osgeo4w.sh index 24a46d16288..fb130a70ff8 100755 --- a/mswindows/osgeo4w/build_osgeo4w.sh +++ b/mswindows/osgeo4w/build_osgeo4w.sh @@ -50,6 +50,7 @@ export ARCH=x86_64-w64-mingw32 --with-postgres-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-gdal=${SRC}/mswindows/osgeo4w/gdal-config \ --with-geos=${SRC}/mswindows/osgeo4w/geos-config \ + --with-libpng=$PWD/mswindows/osgeo4w/libpng-config \ --with-sqlite \ --with-sqlite-includes=${OSGEO4W_ROOT_MSYS}/include \ --with-sqlite-libs=${OSGEO4W_ROOT_MSYS}/lib \ diff --git a/mswindows/osgeo4w/libpng-config b/mswindows/osgeo4w/libpng-config index f00a136be1e..e8d8be2f671 100755 --- a/mswindows/osgeo4w/libpng-config +++ b/mswindows/osgeo4w/libpng-config @@ -14,7 +14,7 @@ prefix="${OSGEO4W_ROOT_MSYS}" version="$(sed '/^#define PNG_LIBPNG_VER_STRING/!d; s/^[^"]*"\|"//g' ${prefix}/include/libpng*/png.h)" -dll_version="$(sed '/^#define PNG_LIBPNG_VER_DLLNUM/!d; s/^[^0-9]*\|[^0-9]*$//g' ${prefix}/include/libpng*/png.h)" +dll_version="$(sed '/^#define PNG_LIBPNG_VER_SHAREDLIB/!d; s/^[^0-9]*\|[^0-9]*$//g' ${prefix}/include/libpng*/png.h)" exec_prefix="${prefix}" libdir="${prefix}/lib" includedir="${prefix}/include/libpng${dll_version}" diff --git a/mswindows/osgeo4w/package.sh b/mswindows/osgeo4w/package.sh index 713396f3cc6..ad8fb659513 100755 --- a/mswindows/osgeo4w/package.sh +++ b/mswindows/osgeo4w/package.sh @@ -115,7 +115,6 @@ DLLS=" /mingw64/bin/libsystre-0.dll /mingw64/bin/libtre-5.dll /mingw64/bin/libwinpthread-1.dll - /mingw64/bin/libpng16-16.dll /mingw64/bin/libgomp-1.dll /mingw64/bin/libomp.dll /mingw64/bin/libstdc++-6.dll @@ -168,6 +167,7 @@ if ! [ -f mswindows/osgeo4w/configure-stamp ]; then --with-postgres-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-gdal=$PWD/mswindows/osgeo4w/gdal-config \ --with-geos=$PWD/mswindows/osgeo4w/geos-config \ + --with-libpng=$PWD/mswindows/osgeo4w/libpng-config \ --with-sqlite \ --with-sqlite-includes=$OSGEO4W_ROOT_MSYS/include \ --with-sqlite-libs=$PWD/mswindows/osgeo4w/lib \ From b43550307c568120c0150a63abb370804c9179a7 Mon Sep 17 00:00:00 2001 From: Martin Landa Date: Thu, 22 Aug 2024 20:57:22 +0200 Subject: [PATCH 07/53] add numpy into deps in order to fix some tests --- .github/workflows/osgeo4w.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index a5302d50dab..d6f450d2cd4 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -55,7 +55,7 @@ jobs: Deps: "gdal-devel,proj-devel,geos-devel,netcdf-devel,libjpeg-turbo-devel,\ libpq-devel,libpng-devel,libtiff-devel,sqlite3-devel,zstd-devel,python3-ply,\ python3-core,python3-six,python3-pywin32,python3-wxpython,liblas-devel,\ - cairo-devel,freetype-devel" + cairo-devel,freetype-devel,python3-numpy" - name: Set number of cores for compilation run: | From 5c4bd6af5a75370fa23637bcd823573cb32cbd57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 21 Sep 2024 14:11:50 +0000 Subject: [PATCH 08/53] CI(OSGeo4W): Sort OSGeo4W packages in a multiline list --- .github/workflows/osgeo4w.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index ef7f2fed2f0..6bb3c299208 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -50,10 +50,25 @@ jobs: with: package-dir: "D:/OSGeo4W_pkg" packages: | - gdal-devel,proj-devel,geos-devel,netcdf-devel,libjpeg-turbo-devel,\ - libpq-devel,libpng-devel,libtiff-devel,sqlite3-devel,zstd-devel,python3-ply,\ - python3-core,python3-six,python3-pywin32,python3-wxpython,liblas-devel,\ - cairo-devel,freetype-devel,python3-numpy + cairo-devel + freetype-devel + gdal-devel + geos-devel + libjpeg-turbo-devel + liblas-devel + libpng-devel + libpq-devel + libtiff-devel + netcdf-devel + proj-devel + python3-core + python3-numpy + python3-ply + python3-pywin32 + python3-six + python3-wxpython + sqlite3-devel + zstd-devel - name: Set number of cores for compilation run: | From ba318835094b5d8a1e2c16b40a4044d72d57d1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 5 Oct 2024 20:42:34 -0400 Subject: [PATCH 09/53] Add back matplotlib package --- .github/workflows/osgeo4w.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 74503ced291..a5733461c19 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -63,6 +63,7 @@ jobs: proj-devel python3-core python3-jupyter + python3-matplotlib python3-numpy python3-pip python3-ply From 6e7532aac91bf50e8fdb33f0b27a7c7292fc85a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 10 Nov 2024 17:44:54 +0000 Subject: [PATCH 10/53] style: Sort packages and configure flags in mswindows build scripts --- mswindows/osgeo4w/build_osgeo4w.sh | 58 +++++++++++----------- mswindows/osgeo4w/package.sh | 79 +++++++++++++++--------------- 2 files changed, 68 insertions(+), 69 deletions(-) diff --git a/mswindows/osgeo4w/build_osgeo4w.sh b/mswindows/osgeo4w/build_osgeo4w.sh index 0c2950237f4..a1c901ff22a 100755 --- a/mswindows/osgeo4w/build_osgeo4w.sh +++ b/mswindows/osgeo4w/build_osgeo4w.sh @@ -22,49 +22,49 @@ export PYTHONHOME=${OSGEO4W_ROOT_MSYS}/apps/Python312 export ARCH=x86_64-w64-mingw32 ./configure \ + --bindir=${OSGEO4W_ROOT_MSYS}/bin \ + --enable-largefile \ + --enable-shared \ --host=${ARCH} \ - --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \ - --with-includes=${OSGEO4W_ROOT_MSYS}/include \ + --includedir=${OSGEO4W_ROOT_MSYS}/include \ --libexecdir=${OSGEO4W_ROOT_MSYS}/bin \ --prefix=${OSGEO4W_ROOT_MSYS}/apps/grass \ - --bindir=${OSGEO4W_ROOT_MSYS}/bin \ - --includedir=${OSGEO4W_ROOT_MSYS}/include \ - --without-x \ + --with-blas \ + --with-bzlib \ + --with-cairo \ + --with-cairo-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-cairo-ldflags="-L${SRC}/mswindows/osgeo4w/lib -lcairo" \ + --with-cairo-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-cxx \ - --enable-shared \ - --enable-largefile \ - --with-openmp \ --with-fftw \ - --with-nls \ - --with-readline \ - --with-blas \ - --with-lapack \ --with-freetype \ --with-freetype-includes=${OSGEO4W_ROOT_MSYS}/include/freetype2 \ - --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \ - --with-proj-includes=${OSGEO4W_ROOT_MSYS}/include \ - --with-proj-libs=${OSGEO4W_ROOT_MSYS}/lib \ - --with-postgres \ - --with-postgres-includes=${OSGEO4W_ROOT_MSYS}/include \ - --with-postgres-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-gdal=${SRC}/mswindows/osgeo4w/gdal-config \ --with-geos=${SRC}/mswindows/osgeo4w/geos-config \ + --with-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-lapack \ + --with-liblas=${SRC}/mswindows/osgeo4w/liblas-config \ --with-libpng=$PWD/mswindows/osgeo4w/libpng-config \ + --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \ + --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config \ + --with-nls \ + --with-odbc \ + --with-opengl=windows \ + --with-openmp \ + --with-postgres \ + --with-postgres-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-postgres-libs=${OSGEO4W_ROOT_MSYS}/lib \ + --with-proj-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-proj-libs=${OSGEO4W_ROOT_MSYS}/lib \ + --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \ + --with-readline \ + --with-regex \ --with-sqlite \ --with-sqlite-includes=${OSGEO4W_ROOT_MSYS}/include \ --with-sqlite-libs=${OSGEO4W_ROOT_MSYS}/lib \ - --with-regex \ --with-zstd \ - --with-odbc \ - --with-cairo \ - --with-cairo-includes=${OSGEO4W_ROOT_MSYS}/include \ - --with-cairo-libs=${OSGEO4W_ROOT_MSYS}/lib \ - --with-cairo-ldflags="-L${SRC}/mswindows/osgeo4w/lib -lcairo" \ - --with-opengl=windows \ - --with-bzlib \ - --with-liblas=${SRC}/mswindows/osgeo4w/liblas-config \ - --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config \ - --without-pdal + --without-pdal \ + --without-x make diff --git a/mswindows/osgeo4w/package.sh b/mswindows/osgeo4w/package.sh index ad8fb659513..a32d65228d1 100755 --- a/mswindows/osgeo4w/package.sh +++ b/mswindows/osgeo4w/package.sh @@ -107,23 +107,23 @@ fi exec 3>&1 > >(tee mswindows/osgeo4w/package.log) 2>&1 DLLS=" - /mingw64/bin/zlib1.dll + /mingw64/bin/libblas.dll /mingw64/bin/libbz2-1.dll - /mingw64/bin/libiconv-2.dll + /mingw64/bin/libfftw3-3.dll + /mingw64/bin/libgcc_s_seh-1.dll /mingw64/bin/libgfortran-5.dll - /mingw64/bin/libintl-8.dll - /mingw64/bin/libsystre-0.dll - /mingw64/bin/libtre-5.dll - /mingw64/bin/libwinpthread-1.dll /mingw64/bin/libgomp-1.dll - /mingw64/bin/libomp.dll - /mingw64/bin/libstdc++-6.dll - /mingw64/bin/libgcc_s_seh-1.dll - /mingw64/bin/libfftw3-3.dll - /mingw64/bin/libblas.dll + /mingw64/bin/libiconv-2.dll + /mingw64/bin/libintl-8.dll /mingw64/bin/liblapack.dll + /mingw64/bin/libomp.dll /mingw64/bin/libquadmath-0.dll /mingw64/bin/libreadline8.dll + /mingw64/bin/libstdc++-6.dll + /mingw64/bin/libsystre-0.dll + /mingw64/bin/libtre-5.dll + /mingw64/bin/libwinpthread-1.dll + /mingw64/bin/zlib1.dll " if ! [ -f mswindows/osgeo4w/configure-stamp ]; then @@ -144,49 +144,48 @@ if ! [ -f mswindows/osgeo4w/configure-stamp ]; then log configure ./configure \ + --bindir=${OSGEO4W_ROOT_MSYS}/bin \ + --enable-largefile \ + --enable-shared \ --host=x86_64-w64-mingw32 \ - --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \ - --with-includes=${OSGEO4W_ROOT_MSYS}/include \ + --includedir=${OSGEO4W_ROOT_MSYS}/include \ --libexecdir=${OSGEO4W_ROOT_MSYS}/bin \ --prefix=${OSGEO4W_ROOT_MSYS}/apps/grass \ - --bindir=${OSGEO4W_ROOT_MSYS}/bin \ - --includedir=${OSGEO4W_ROOT_MSYS}/include \ - --with-opengl=windows \ - --without-x \ + --with-blas \ + --with-bzlib \ + --with-cairo \ + --with-cairo-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-cairo-ldflags="-L$PWD/mswindows/osgeo4w/lib -lcairo" \ --with-cxx \ - --enable-shared \ - --enable-largefile \ --with-fftw \ --with-freetype \ --with-freetype-includes=${OSGEO4W_ROOT_MSYS}/include/freetype2 \ - --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \ - --with-proj-includes=${OSGEO4W_ROOT_MSYS}/include \ - --with-proj-libs=${OSGEO4W_ROOT_MSYS}/lib \ + --with-gdal=$PWD/mswindows/osgeo4w/gdal-config \ + --with-geos=$PWD/mswindows/osgeo4w/geos-config \ + --with-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-lapack \ + --with-lapack-includes=/mingw64/include \ + --with-liblas=$PWD/mswindows/osgeo4w/liblas-config \ + --with-libpng=$PWD/mswindows/osgeo4w/libpng-config \ + --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \ + --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config \ + --with-nls \ + --with-odbc \ + --with-opengl=windows \ + --with-openmp \ --with-postgres \ --with-postgres-includes=${OSGEO4W_ROOT_MSYS}/include \ --with-postgres-libs=${OSGEO4W_ROOT_MSYS}/lib \ - --with-gdal=$PWD/mswindows/osgeo4w/gdal-config \ - --with-geos=$PWD/mswindows/osgeo4w/geos-config \ - --with-libpng=$PWD/mswindows/osgeo4w/libpng-config \ + --with-proj-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-proj-libs=${OSGEO4W_ROOT_MSYS}/lib \ + --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \ + --with-regex \ --with-sqlite \ --with-sqlite-includes=$OSGEO4W_ROOT_MSYS/include \ --with-sqlite-libs=$PWD/mswindows/osgeo4w/lib \ - --with-regex \ - --with-nls \ --with-zstd \ - --with-odbc \ - --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config \ - --with-blas \ - --with-lapack \ - --with-lapack-includes=/mingw64/include \ - --with-openmp \ - --with-cairo \ - --with-cairo-includes=${OSGEO4W_ROOT_MSYS}/include \ - --with-cairo-ldflags="-L$PWD/mswindows/osgeo4w/lib -lcairo" \ - --with-bzlib \ - --with-liblas=$PWD/mswindows/osgeo4w/liblas-config \ - --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config \ - --without-pdal + --without-pdal \ + --without-x touch mswindows/osgeo4w/configure-stamp fi From 2f9c87ac0349648116ea4b6a34928d527d3f2308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 23 Nov 2024 14:03:21 +0000 Subject: [PATCH 11/53] packaging: Removed changes from patch that was removed upstream https://github.com/jef-n/OSGeo4W/commit/c678c71d080e5c33e8cb64b12386be2ca4b6c54a --- include/Make/Grass.make | 2 +- include/Make/Platform.make.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/Make/Grass.make b/include/Make/Grass.make index 8d2476de028..17ce25fcb71 100644 --- a/include/Make/Grass.make +++ b/include/Make/Grass.make @@ -198,7 +198,7 @@ DSPFDEPS = $(GISLIB) FORMDEPS = $(DBMILIB) $(GISLIB) RASTER3DDEPS = $(RASTERLIB) $(GISLIB) GISDEPS = $(DATETIMELIB) $(ZLIBLIBPATH) $(ZLIB) $(BZIP2LIBPATH) $(BZIP2LIB) $(ZSTDLIBPATH) $(ZSTDLIB) $(INTLLIB) $(REGEXLIBPATH) $(REGEXLIB) $(ICONVLIB) $(PTHREADLIBPATH) $(PTHREADLIB) $(MATHLIB) -GMATHDEPS = $(GISLIB) $(FFTWLIB) $(LAPACKLIB) $(BLASLIB) $(CCMATHLIB) $(OPENMP_LIBPATH) $(OPENMP_LIB) +GMATHDEPS = $(GISLIB) $(FFTWLIB) $(LAPACKLIB) $(BLASLIB) $(CCMATHLIB) $(OPENMP_CFLAGS) $(OPENMP_LIBPATH) $(OPENMP_LIB) GPDEDEPS = $(RASTER3DLIB) $(RASTERLIB) $(GISLIB) $(GMATHLIB) $(OPENMP_LIBPATH) $(OPENMP_LIB) $(MATHLIB) GPROJDEPS = $(GISLIB) $(GDALLIBS) $(PROJLIB) $(MATHLIB) HTMLDRIVERDEPS = $(DRIVERLIB) $(GISLIB) $(MATHLIB) diff --git a/include/Make/Platform.make.in b/include/Make/Platform.make.in index 1f08f77719a..a66ddf801d3 100644 --- a/include/Make/Platform.make.in +++ b/include/Make/Platform.make.in @@ -228,7 +228,7 @@ USE_PTHREAD = @USE_PTHREAD@ #OpenMP OPENMP_INCPATH = @OPENMP_INCPATH@ OPENMP_LIBPATH = @OPENMP_LIBPATH@ -OPENMP_LIB = @OPENMP_CFLAGS@ @OPENMP_LIB@ +OPENMP_LIB = @OPENMP_LIB@ OPENMP_CFLAGS = @OPENMP_CFLAGS@ USE_OPENMP = @USE_OPENMP@ From b6bd490cf01b09fc6d7bb56ca8388a4c9ae5454e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 23 Nov 2024 14:55:30 +0000 Subject: [PATCH 12/53] packaging: Apply upstream patch update https://github.com/jef-n/OSGeo4W/commit/4bd721f07ac8036274328a9e4500c1fa944a0926 --- .github/workflows/osgeo4w.yml | 2 -- mswindows/osgeo4w/build_osgeo4w.sh | 2 ++ mswindows/osgeo4w/package.sh | 31 +++++++++++++++--------------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 1f603ae6e6e..3e9af865073 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -82,8 +82,6 @@ jobs: - name: Compile GRASS GIS shell: msys2 {0} run: | - export CFLAGS="${CFLAGS} -pipe" - export CXXFLAGS="${CXXFLAGS} -pipe" .github/workflows/build_osgeo4w.sh - name: Print installed versions diff --git a/mswindows/osgeo4w/build_osgeo4w.sh b/mswindows/osgeo4w/build_osgeo4w.sh index a1c901ff22a..c4209b38c2b 100755 --- a/mswindows/osgeo4w/build_osgeo4w.sh +++ b/mswindows/osgeo4w/build_osgeo4w.sh @@ -21,6 +21,8 @@ export C_INCLUDE_PATH=".:${OSGEO4W_ROOT_MSYS}/include:${SRC}/dist.${ARCH}/includ export PYTHONHOME=${OSGEO4W_ROOT_MSYS}/apps/Python312 export ARCH=x86_64-w64-mingw32 +CFLAGS="$CFLAGS -pipe" \ +CXXFLAGS="$CXXFLAGS -pipe" \ ./configure \ --bindir=${OSGEO4W_ROOT_MSYS}/bin \ --enable-largefile \ diff --git a/mswindows/osgeo4w/package.sh b/mswindows/osgeo4w/package.sh index 2406339e458..bfd71cfa31a 100755 --- a/mswindows/osgeo4w/package.sh +++ b/mswindows/osgeo4w/package.sh @@ -1,8 +1,9 @@ -#!/usr/bin/bash +#!/bin/bash set -e -PWD="$(pwd)" +export ARCH=x86_64-w64-mingw32 +export SRC=$PWD if ! [ -d mswindows ]; then echo Start from GRASS toplevel dir @@ -136,19 +137,18 @@ if ! [ -f mswindows/osgeo4w/configure-stamp ]; then log remove old logs rm -f mswindows/osgeo4w/package.log.* - mkdir -p dist.x86_64-w64-mingw32/bin - cp -uv $DLLS dist.x86_64-w64-mingw32/bin - mkdir -p mswindows/osgeo4w/lib cp -uv $OSGEO4W_ROOT_MSYS/lib/sqlite3_i.lib mswindows/osgeo4w/lib/sqlite3.lib log configure + CFLAGS="$CFLAGS -pipe" \ + CXXFLAGS="$CXXFLAGS -pipe" \ ./configure \ --bindir=${OSGEO4W_ROOT_MSYS}/bin \ --enable-largefile \ --enable-shared \ - --host=x86_64-w64-mingw32 \ + --host=${ARCH} \ --includedir=${OSGEO4W_ROOT_MSYS}/include \ --libexecdir=${OSGEO4W_ROOT_MSYS}/bin \ --prefix=${OSGEO4W_ROOT_MSYS}/apps/grass \ @@ -156,17 +156,17 @@ if ! [ -f mswindows/osgeo4w/configure-stamp ]; then --with-bzlib \ --with-cairo \ --with-cairo-includes=${OSGEO4W_ROOT_MSYS}/include \ - --with-cairo-ldflags="-L$PWD/mswindows/osgeo4w/lib -lcairo" \ + --with-cairo-ldflags="-L${SRC}/mswindows/osgeo4w/lib -lcairo" \ + --with-cairo-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-cxx \ --with-fftw \ --with-freetype \ --with-freetype-includes=${OSGEO4W_ROOT_MSYS}/include/freetype2 \ - --with-gdal=$PWD/mswindows/osgeo4w/gdal-config \ - --with-geos=$PWD/mswindows/osgeo4w/geos-config \ + --with-gdal=${SRC}/mswindows/osgeo4w/gdal-config \ + --with-geos=${SRC}/mswindows/osgeo4w/geos-config \ --with-includes=${OSGEO4W_ROOT_MSYS}/include \ --with-lapack \ - --with-liblas=$PWD/mswindows/osgeo4w/liblas-config \ - --with-libpng=$PWD/mswindows/osgeo4w/libpng-config \ + --with-liblas=${SRC}/mswindows/osgeo4w/liblas-config \ --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \ --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config \ --with-nls \ @@ -179,10 +179,11 @@ if ! [ -f mswindows/osgeo4w/configure-stamp ]; then --with-proj-includes=${OSGEO4W_ROOT_MSYS}/include \ --with-proj-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-proj-share=${OSGEO4W_ROOT_MSYS}/share/proj \ + --with-readline \ --with-regex \ --with-sqlite \ - --with-sqlite-includes=$OSGEO4W_ROOT_MSYS/include \ - --with-sqlite-libs=$PWD/mswindows/osgeo4w/lib \ + --with-sqlite-includes=${OSGEO4W_ROOT_MSYS}/include \ + --with-sqlite-libs=${OSGEO4W_ROOT_MSYS}/lib \ --with-zstd \ --without-pdal \ --without-x @@ -238,8 +239,8 @@ if [ -n "$PACKAGE_PATCH" ]; then unix2dos etc/postinstall/grass${PACKAGE_POSTFIX}.bat unix2dos etc/preremove/grass${PACKAGE_POSTFIX}.bat - # copy dependencies (TODO: to be reduced) - cp -uv $DLLS apps/grass/grass$POSTFIX/bin + # copy dependencies + cp -uv $(/usr/bin/find apps/grass/grass$POSTFIX -iname "*.dll" -o -iname "*.exe" | PATH=$PWD/apps/grass/grass$POSTFIX/lib:$PWD/bin:/mingw64/bin:/usr/bin /usr/bin/xargs /usr/bin/ldd | /usr/bin/sed -ne 's#^.* => \(/mingw64/bin/.*\) (.*)$#\1#p' | /usr/bin/sort -u) apps/grass/grass$POSTFIX/bin # copy R batch files cp -uv $SRC/mswindows/external/rbatch/* apps/grass/grass$POSTFIX/bin From 7ffcbadcca5be3600690b39cf4e9b0f5ae530497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sat, 23 Nov 2024 14:56:39 +0000 Subject: [PATCH 13/53] packaging: Keep --with-libpng with libpng-config from repo --- mswindows/osgeo4w/build_osgeo4w.sh | 2 +- mswindows/osgeo4w/package.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mswindows/osgeo4w/build_osgeo4w.sh b/mswindows/osgeo4w/build_osgeo4w.sh index c4209b38c2b..252b615339f 100755 --- a/mswindows/osgeo4w/build_osgeo4w.sh +++ b/mswindows/osgeo4w/build_osgeo4w.sh @@ -46,7 +46,7 @@ CXXFLAGS="$CXXFLAGS -pipe" \ --with-includes=${OSGEO4W_ROOT_MSYS}/include \ --with-lapack \ --with-liblas=${SRC}/mswindows/osgeo4w/liblas-config \ - --with-libpng=$PWD/mswindows/osgeo4w/libpng-config \ + --with-libpng=${SRC}/mswindows/osgeo4w/libpng-config \ --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \ --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config \ --with-nls \ diff --git a/mswindows/osgeo4w/package.sh b/mswindows/osgeo4w/package.sh index bfd71cfa31a..2d62553a5c8 100755 --- a/mswindows/osgeo4w/package.sh +++ b/mswindows/osgeo4w/package.sh @@ -167,6 +167,7 @@ if ! [ -f mswindows/osgeo4w/configure-stamp ]; then --with-includes=${OSGEO4W_ROOT_MSYS}/include \ --with-lapack \ --with-liblas=${SRC}/mswindows/osgeo4w/liblas-config \ + --with-libpng=${SRC}/mswindows/osgeo4w/libpng-config \ --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \ --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config \ --with-nls \ From c03abb9424ff003bfa5d4a6f831d854e71a5ba95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 11 Nov 2024 22:29:44 -0500 Subject: [PATCH 14/53] Compile and upload package using package.sh --- .github/workflows/osgeo4w.yml | 48 +++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 3e9af865073..5c454802cb8 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -73,7 +73,26 @@ jobs: python3-wxpython sqlite3-devel zstd-devel - + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + shell: cmake -P {0} + run: | + string(TIMESTAMP current_date "%Y-%m-%d-%H-%M-%S" UTC) + file(APPEND "$ENV{GITHUB_OUTPUT}" "timestamp=${current_date}") + - name: ccache cache files + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: .ccache + key: ${{ matrix.os }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + restore-keys: | + ${{ matrix.os }}-ccache- + save-always: true + - name: Configure ccache + run: | + echo "CCACHE_BASEDIR=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}" + echo "CCACHE_DIR=${GITHUB_WORKSPACE}/.ccache" >> "${GITHUB_ENV}" + echo "CCACHE_MAXSIZE=400M" >> "${GITHUB_ENV}" + shell: msys2 {0} - name: Set number of cores for compilation run: | echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV @@ -82,7 +101,26 @@ jobs: - name: Compile GRASS GIS shell: msys2 {0} run: | - .github/workflows/build_osgeo4w.sh + ccache --show-stats + MSYSPATH=$(cygpath -a msys64/usr/bin):$OSGEO4W_ROOT_MSYS/bin:$(cygpath --sysdir) + MSYSPATH=${MSYSPATH//\/cygdrive/} + ccache --show-config + ccache --zero-stats + PATH=$MSYSPATH:$PATH mswindows/osgeo4w/package.sh + ccache --show-stats --verbose --verbose + env: + PACKAGE_POSTFIX: "-dev" + OSGEO4W_ROOT: "C:\\OSGeo4W" + OSGEO4W_ROOT_MSYS: "/c/OSGeo4W" + PACKAGE_PATCH: ${{ github.sha }}-1 + - name: Upload OSGeo4W grass package artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: grass-osgeo4w-package-${{ matrix.os }} + path: mswindows/osgeo4w/package + include-hidden-files: true + retention-days: 3 - name: Print installed versions if: always() @@ -90,7 +128,7 @@ jobs: run: .github/workflows/print_versions.sh - name: Test executing of the grass command - run: .github/workflows/test_simple.bat 'C:\OSGeo4W\opt\grass\grass85.bat' + run: .github/workflows/test_simple.bat 'C:\OSGeo4W\bin\grass85.bat' - name: Test executing of the grass command in bash shell: msys2 {0} @@ -101,7 +139,7 @@ jobs: shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}"" - name: Run pytest with a single worker run: | - call %OSGEO4W_ROOT%\opt\grass\etc\env.bat + call %OSGEO4W_ROOT%\apps\grass\grass85\etc\env.bat set PYTHONPATH=%GISBASE%\etc\python;%PYTHONPATH% path %GISBASE%\lib;%GISBASE%\bin;%PATH% pytest --verbose --color=yes ^ @@ -110,7 +148,7 @@ jobs: shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}"" - name: Run tests - run: .github/workflows/test_thorough.bat 'C:\OSGeo4W\opt\grass\grass85.bat' 'C:\OSGeo4W\bin\python3' + run: .github/workflows/test_thorough.bat 'C:\OSGeo4W\bin\grass85.bat' 'C:\OSGeo4W\bin\python3' - name: Make HTML test report available if: ${{ always() }} From 991ec8e316dadd45c584be7d061b931e5c9db7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 11 Nov 2024 23:20:44 -0500 Subject: [PATCH 15/53] Use mingw-w64-x86_64-omp --- .github/workflows/osgeo4w.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 5c454802cb8..6b30a348919 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -43,7 +43,7 @@ jobs: mingw-w64-x86_64-gcc mingw-w64-x86_64-ccache mingw-w64-x86_64-zlib mingw-w64-x86_64-libiconv mingw-w64-x86_64-bzip2 mingw-w64-x86_64-gettext mingw-w64-x86_64-libsystre mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git - mingw-w64-x86_64-pcre + mingw-w64-x86_64-pcre mingw-w64-x86_64-omp - name: Setup OSGeo4W environment uses: echoix/setup-OSGeo4W@17deecd39e077a80bf1081443998ea8edd6f15bf # v0.1.0 From 6b6b35a26fb500e57ac12322d4a27b1287930d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Mon, 11 Nov 2024 23:35:37 -0500 Subject: [PATCH 16/53] Add mingw-w64-x86_64-lapack package --- .github/workflows/osgeo4w.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 6b30a348919..d8410eb90f2 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -43,7 +43,7 @@ jobs: mingw-w64-x86_64-gcc mingw-w64-x86_64-ccache mingw-w64-x86_64-zlib mingw-w64-x86_64-libiconv mingw-w64-x86_64-bzip2 mingw-w64-x86_64-gettext mingw-w64-x86_64-libsystre mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git - mingw-w64-x86_64-pcre mingw-w64-x86_64-omp + mingw-w64-x86_64-pcre mingw-w64-x86_64-omp mingw-w64-x86_64-lapack - name: Setup OSGeo4W environment uses: echoix/setup-OSGeo4W@17deecd39e077a80bf1081443998ea8edd6f15bf # v0.1.0 From fc2e53af6b0a65742a70ccfa2f0920aca6f05745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Tue, 12 Nov 2024 06:47:55 -0500 Subject: [PATCH 17/53] Do not fail step print_version.sh missing msys2 paths --- .github/workflows/osgeo4w.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index d8410eb90f2..17d8ef8be49 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -126,6 +126,8 @@ jobs: if: always() shell: msys2 {0} run: .github/workflows/print_versions.sh + # TODO: find the step that adds back msys2 env into a file loaded by osgeo4w shell + continue-on-error: true - name: Test executing of the grass command run: .github/workflows/test_simple.bat 'C:\OSGeo4W\bin\grass85.bat' From 82fc6f9a430ac837038dd68d673db467e14ad624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Tue, 12 Nov 2024 07:08:39 -0500 Subject: [PATCH 18/53] Update osgeo4w workflow to continue on error --- .github/workflows/osgeo4w.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 17d8ef8be49..669b790344d 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -135,6 +135,7 @@ jobs: - name: Test executing of the grass command in bash shell: msys2 {0} run: .github/workflows/test_simple.sh + continue-on-error: true # TODO - name: Install pytest plugins run: python -m pip install pytest-timeout From c1d2beb718c0f17ca841e973c7a79f746b295201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 01:11:51 +0000 Subject: [PATCH 19/53] Add msys to paths loaded by OSGeo4W --- .github/workflows/osgeo4w.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 669b790344d..9e8bf8f2356 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -33,6 +33,7 @@ jobs: git config --global core.eol lf - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: msys2/setup-msys2@c52d1fa9c7492275e60fe763540fb601f5f232a1 # v2.25.0 + id: msys2 with: path-type: inherit location: D:\ @@ -122,6 +123,16 @@ jobs: include-hidden-files: true retention-days: 3 + - name: Add msys to paths loaded by OSGeo4W + run: | + # $msys_path=";C:/msys64/usr/bin;C:/msys64/mingw64/bin" + $msys_path=";$env:MSYS2_ROOT/usr/bin;$env:MSYS2_ROOT/mingw64/bin" + $line="set PATH=%OSGEO4W_ROOT%\bin${msys_path};%PATH%" + $line | Out-File -FilePath $env:OSGEO4W_ROOT/etc/ini/z_msys.bat + env: + MSYS2_ROOT: ${{ steps.msys2.outputs.msys2-location }} + OSGEO4W_ROOT: "C:\\OSGeo4W" + continue-on-error: true - name: Print installed versions if: always() shell: msys2 {0} From 68ae07f7973d036ddfde971f08c2ff7efbc1a5d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 01:27:26 +0000 Subject: [PATCH 20/53] Use root output from setup-OSGeo4W --- .github/workflows/osgeo4w.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 9e8bf8f2356..33a2fd250e7 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -47,7 +47,8 @@ jobs: mingw-w64-x86_64-pcre mingw-w64-x86_64-omp mingw-w64-x86_64-lapack - name: Setup OSGeo4W environment - uses: echoix/setup-OSGeo4W@17deecd39e077a80bf1081443998ea8edd6f15bf # v0.1.0 + uses: echoix/setup-OSGeo4W@5b90433c1c502945f914cef45735fa9e15563840 # v0.2.0 + id: osgeo4w with: package-dir: "D:/OSGeo4W_pkg" packages: | @@ -111,7 +112,7 @@ jobs: ccache --show-stats --verbose --verbose env: PACKAGE_POSTFIX: "-dev" - OSGEO4W_ROOT: "C:\\OSGeo4W" + OSGEO4W_ROOT: ${{ steps.osgeo4w.outputs.root }} OSGEO4W_ROOT_MSYS: "/c/OSGeo4W" PACKAGE_PATCH: ${{ github.sha }}-1 - name: Upload OSGeo4W grass package artifact @@ -125,13 +126,12 @@ jobs: - name: Add msys to paths loaded by OSGeo4W run: | - # $msys_path=";C:/msys64/usr/bin;C:/msys64/mingw64/bin" $msys_path=";$env:MSYS2_ROOT/usr/bin;$env:MSYS2_ROOT/mingw64/bin" $line="set PATH=%OSGEO4W_ROOT%\bin${msys_path};%PATH%" $line | Out-File -FilePath $env:OSGEO4W_ROOT/etc/ini/z_msys.bat env: MSYS2_ROOT: ${{ steps.msys2.outputs.msys2-location }} - OSGEO4W_ROOT: "C:\\OSGeo4W" + OSGEO4W_ROOT: ${{ steps.osgeo4w.outputs.root }} continue-on-error: true - name: Print installed versions if: always() @@ -162,7 +162,7 @@ jobs: shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}"" - name: Run tests - run: .github/workflows/test_thorough.bat 'C:\OSGeo4W\bin\grass85.bat' 'C:\OSGeo4W\bin\python3' + run: .github/workflows/test_thorough.bat '${{ steps.osgeo4w.outputs.root }}\bin\grass85.bat' '${{ steps.osgeo4w.outputs.root }}\bin\python3' - name: Make HTML test report available if: ${{ always() }} From 092c7693ce43897f3dd07ae9b5860ec289262eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 01:31:53 +0000 Subject: [PATCH 21/53] Fix long lines with yaml line folding --- .github/workflows/osgeo4w.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 33a2fd250e7..e30f1531ffb 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -13,7 +13,8 @@ jobs: name: ${{ matrix.os }} build and tests concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}-${{ + group: >- + ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}-${{ matrix.os }} cancel-in-progress: true @@ -39,7 +40,8 @@ jobs: location: D:\ update: true msystem: MINGW64 - install: tar libintl make bison flex diffutils git dos2unix zip mingw-w64-x86_64-toolchain + install: >- + tar libintl make bison flex diffutils git dos2unix zip mingw-w64-x86_64-toolchain mingw-w64-x86_64-fftw mingw-w64-x86_64-openblas mingw-w64-x86_64-pkgconf mingw-w64-x86_64-gcc mingw-w64-x86_64-ccache mingw-w64-x86_64-zlib mingw-w64-x86_64-libiconv mingw-w64-x86_64-bzip2 mingw-w64-x86_64-gettext mingw-w64-x86_64-libsystre From 82c41973dd4ba9732f43b648edf9a2118aa284df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 01:32:12 +0000 Subject: [PATCH 22/53] Fix remaining formatting issues --- .github/workflows/osgeo4w.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index e30f1531ffb..a4cb7566390 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -79,12 +79,12 @@ jobs: zstd-devel - name: Prepare ccache timestamp id: ccache_cache_timestamp - shell: cmake -P {0} + shell: cmake -P {0} run: | string(TIMESTAMP current_date "%Y-%m-%d-%H-%M-%S" UTC) file(APPEND "$ENV{GITHUB_OUTPUT}" "timestamp=${current_date}") - name: ccache cache files - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: .ccache key: ${{ matrix.os }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} From bfc5f787d3de3ef545739b758aebf3d5da9c1cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 01:45:12 +0000 Subject: [PATCH 23/53] Fix line too long --- .github/workflows/osgeo4w.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index a4cb7566390..e876b2e43e4 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -164,8 +164,9 @@ jobs: shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}"" - name: Run tests - run: .github/workflows/test_thorough.bat '${{ steps.osgeo4w.outputs.root }}\bin\grass85.bat' '${{ steps.osgeo4w.outputs.root }}\bin\python3' - + run: .github/workflows/test_thorough.bat '$env:O4WROOT\bin\grass85.bat' '$env:O4WROOT\bin\python3' + env: + O4WROOT: ${{ steps.osgeo4w.outputs.root }} - name: Make HTML test report available if: ${{ always() }} uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 From e096e41bbcde53b8112cb60acf1eed4fe213078f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 01:55:07 +0000 Subject: [PATCH 24/53] Add env vars for print versions in msys2 bash works correctly --- .github/workflows/osgeo4w.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index e876b2e43e4..80c5d0aa3c3 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -138,8 +138,15 @@ jobs: - name: Print installed versions if: always() shell: msys2 {0} - run: .github/workflows/print_versions.sh - # TODO: find the step that adds back msys2 env into a file loaded by osgeo4w shell + run: | + export ARCH=x86_64-w64-mingw32 + PWD="$(pwd)" + SRC="${PWD}" + export PATH="${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin:${SRC}/dist.${ARCH}/bin:${OSGEO4W_ROOT_MSYS}/apps/grass/grass85/etc:${PATH}" + export PYTHONHOME=${OSGEO4W_ROOT_MSYS}/apps/Python312 + .github/workflows/print_versions.sh + env: + OSGEO4W_ROOT_MSYS: "/c/OSGeo4W" continue-on-error: true - name: Test executing of the grass command From 06c5d13cdd81be93ac81596cb7ee4b88ebcbbcb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 02:38:50 +0000 Subject: [PATCH 25/53] Update setup-OSGeo4W action --- .github/workflows/osgeo4w.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 80c5d0aa3c3..1639d0d845b 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -49,7 +49,7 @@ jobs: mingw-w64-x86_64-pcre mingw-w64-x86_64-omp mingw-w64-x86_64-lapack - name: Setup OSGeo4W environment - uses: echoix/setup-OSGeo4W@5b90433c1c502945f914cef45735fa9e15563840 # v0.2.0 + uses: echoix/setup-OSGeo4W@f4311523e39f2c8b10e34ebbc3f2ff437ecfb9ed # v0.2.0 id: osgeo4w with: package-dir: "D:/OSGeo4W_pkg" From f3af06613103d0f8c04216d6a5960d4dd0a3f1ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 02:45:25 +0000 Subject: [PATCH 26/53] Use double quotes --- .github/workflows/osgeo4w.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 1639d0d845b..724b15c229c 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -171,7 +171,7 @@ jobs: shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}"" - name: Run tests - run: .github/workflows/test_thorough.bat '$env:O4WROOT\bin\grass85.bat' '$env:O4WROOT\bin\python3' + run: .github/workflows/test_thorough.bat "$env:O4WROOT\bin\grass85.bat" "$env:O4WROOT\bin\python3" env: O4WROOT: ${{ steps.osgeo4w.outputs.root }} - name: Make HTML test report available From c35ac962da68d636016a639508abf39c9387f29b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 13:30:50 +0000 Subject: [PATCH 27/53] CI(OSGeo4W): Test executing of the grass command in bash msys --- .github/workflows/osgeo4w.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 724b15c229c..08bb8070f10 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -156,6 +156,23 @@ jobs: shell: msys2 {0} run: .github/workflows/test_simple.sh continue-on-error: true # TODO + - name: Test executing of the grass command in bash msys + shell: msys2 {0} + run: | + PWD="$(pwd)" + SRC="${PWD}" + export ARCH=x86_64-w64-mingw32 + export PATH="${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin:${SRC}/dist.${ARCH}/bin:${SRC}/dist.${ARCH}/$bash_bin:${OSGEO4W_ROOT_MSYS}/apps/grass/grass85/etc" + export PYTHONHOME=${OSGEO4W_ROOT_MSYS}/apps/Python312 + export PYTHONHOME="${PYTHONHOME}" + export PYTHONUTF8=1 + export GISBASE=${OSGEO4W_ROOT}/apps/grass/grass85 + printenv | sort + .github/workflows/test_simple_msys.sh + continue-on-error: true + env: + OSGEO4W_ROOT_MSYS: "/c/OSGeo4W" + OSGEO4W_ROOT: ${{ steps.osgeo4w.outputs.root }} - name: Install pytest plugins run: python -m pip install pytest-timeout From c17a100b757ecabd1036ba1dde711465be5040c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 13:50:44 +0000 Subject: [PATCH 28/53] CI(OSGeo4W): Restore and save ccache explicitly --- .github/workflows/osgeo4w.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 08bb8070f10..23c75ad1f87 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -83,14 +83,14 @@ jobs: run: | string(TIMESTAMP current_date "%Y-%m-%d-%H-%M-%S" UTC) file(APPEND "$ENV{GITHUB_OUTPUT}" "timestamp=${current_date}") - - name: ccache cache files - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + - name: Restore ccache cache files + uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + id: restore-cache with: path: .ccache key: ${{ matrix.os }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} restore-keys: | ${{ matrix.os }}-ccache- - save-always: true - name: Configure ccache run: | echo "CCACHE_BASEDIR=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}" @@ -125,7 +125,11 @@ jobs: path: mswindows/osgeo4w/package include-hidden-files: true retention-days: 3 - + - name: Save ccache early + uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: .ccache + key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Add msys to paths loaded by OSGeo4W run: | $msys_path=";$env:MSYS2_ROOT/usr/bin;$env:MSYS2_ROOT/mingw64/bin" From bab6d3a6af9a76d62a1fcef0393f4f79ce90edbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 14:06:50 +0000 Subject: [PATCH 29/53] pygrass: Split hanging modules/interface test into separate files to find which part is problematic --- .../interface/testsuite/_test_modules.py.bak | 70 +++++++++++++++++++ .../interface/testsuite/test_modules_check.py | 33 +++++++++ .../testsuite/test_modules_pickability.py | 31 ++++++++ ...ules.py => test_modules_with_metaclass.py} | 25 ------- 4 files changed, 134 insertions(+), 25 deletions(-) create mode 100644 python/grass/pygrass/modules/interface/testsuite/_test_modules.py.bak create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_check.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_pickability.py rename python/grass/pygrass/modules/interface/testsuite/{test_modules.py => test_modules_with_metaclass.py} (53%) diff --git a/python/grass/pygrass/modules/interface/testsuite/_test_modules.py.bak b/python/grass/pygrass/modules/interface/testsuite/_test_modules.py.bak new file mode 100644 index 00000000000..34c0f03d64c --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/_test_modules.py.bak @@ -0,0 +1,70 @@ +# """ +# Created on Tue Jun 24 09:43:53 2014 + +# @author: pietro +# """ + +# from fnmatch import fnmatch +# from io import BytesIO + +# from grass.gunittest.case import TestCase +# from grass.gunittest.main import test + +# from grass.script.core import get_commands +# from grass.exceptions import ParameterError +# from grass.pygrass.modules.interface import Module + + +# SKIP = [ +# "g.parser", +# ] + + +# class ModulesMeta(type): +# def __new__(cls, name, bases, dict): +# def gen_test(cmd): +# def test(self): +# Module(cmd) + +# return test + +# cmds = [ +# c +# for c in sorted(list(get_commands()[0])) +# if c not in SKIP and not fnmatch(c, "g.gui.*") +# ] +# for cmd in cmds: +# test_name = "test__%s" % cmd.replace(".", "_") +# dict[test_name] = gen_test(cmd) +# return type.__new__(cls, name, bases, dict) + + +# class TestModules(TestCase, metaclass=ModulesMeta): +# pass + + +# class TestModulesPickability(TestCase): +# def test_rsun(self): +# """Test if a Module instance is pickable""" +# import pickle + +# out = BytesIO() +# pickle.dump(Module("r.sun"), out) +# out.close() + + +# class TestModulesCheck(TestCase): +# def test_flags_with_suppress_required(self): +# """Test if flags with suppress required are handle correctly""" +# gextension = Module("g.extension") +# # check if raise an error if required parameter are missing +# with self.assertRaises(ParameterError): +# gextension.check() + +# # check if the flag suppress the required parameters +# gextension.flags.a = True +# self.assertIsNone(gextension.check()) + + +# if __name__ == "__main__": +# test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_check.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_check.py new file mode 100644 index 00000000000..3fdce29500f --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_check.py @@ -0,0 +1,33 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.exceptions import ParameterError +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class TestModulesCheck(TestCase): + def test_flags_with_suppress_required(self): + """Test if flags with suppress required are handle correctly""" + gextension = Module("g.extension") + # check if raise an error if required parameter are missing + with self.assertRaises(ParameterError): + gextension.check() + + # check if the flag suppress the required parameters + gextension.flags.a = True + self.assertIsNone(gextension.check()) + + +if __name__ == "__main__": + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_pickability.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_pickability.py new file mode 100644 index 00000000000..3ac3f1a5074 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_pickability.py @@ -0,0 +1,31 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +from io import BytesIO + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class TestModulesPickability(TestCase): + def test_rsun(self): + """Test if a Module instance is pickable""" + import pickle + + out = BytesIO() + pickle.dump(Module("r.sun"), out) + out.close() + + +if __name__ == "__main__": + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py similarity index 53% rename from python/grass/pygrass/modules/interface/testsuite/test_modules.py rename to python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py index ee005498692..ff528d9c4b3 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py @@ -5,13 +5,11 @@ """ from fnmatch import fnmatch -from io import BytesIO from grass.gunittest.case import TestCase from grass.gunittest.main import test from grass.script.core import get_commands -from grass.exceptions import ParameterError from grass.pygrass.modules.interface import Module @@ -43,28 +41,5 @@ class TestModules(TestCase, metaclass=ModulesMeta): pass -class TestModulesPickability(TestCase): - def test_rsun(self): - """Test if a Module instance is pickable""" - import pickle - - out = BytesIO() - pickle.dump(Module("r.sun"), out) - out.close() - - -class TestModulesCheck(TestCase): - def test_flags_with_suppress_required(self): - """Test if flags with suppress required are handle correctly""" - gextension = Module("g.extension") - # check if raise an error if required parameter are missing - with self.assertRaises(ParameterError): - gextension.check() - - # check if the flag suppress the required parameters - gextension.flags.a = True - self.assertIsNone(gextension.check()) - - if __name__ == "__main__": test() From 85cea88e0cbfff2d2e078f3f45660381459053ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 14:53:29 +0000 Subject: [PATCH 30/53] pygrass: Print statements to track ModulesMeta hangs in modules/interface tests --- .../modules/interface/testsuite/test_modules_with_metaclass.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py index ff528d9c4b3..14ffe0bb959 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py @@ -20,6 +20,8 @@ class ModulesMeta(type): def __new__(cls, name, bases, dict): + print("In __new__ of ModulesMeta") + def gen_test(cmd): def test(self): Module(cmd) @@ -33,6 +35,7 @@ def test(self): ] for cmd in cmds: test_name = "test__%s" % cmd.replace(".", "_") + print(f"cmd is: {cmd}, test_name: {test_name}") dict[test_name] = gen_test(cmd) return type.__new__(cls, name, bases, dict) From f5902882587d1b85d539d2fc6ade6fbf54a5b7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 14:58:37 +0000 Subject: [PATCH 31/53] Replace hardcoded OSGeo4W installed root --- .github/workflows/osgeo4w.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 23c75ad1f87..f13870a0842 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -154,7 +154,9 @@ jobs: continue-on-error: true - name: Test executing of the grass command - run: .github/workflows/test_simple.bat 'C:\OSGeo4W\bin\grass85.bat' + run: .github/workflows/test_simple.bat '${{ env.O4WROOT }}\bin\grass85.bat' + env: + O4WROOT: ${{ steps.osgeo4w.outputs.root }} - name: Test executing of the grass command in bash shell: msys2 {0} From 9a5e8f34319a684bbbf17d7452b56af0dfbe899e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 15:26:27 +0000 Subject: [PATCH 32/53] pygrass: Use logging warnings to track ModulesMeta hangs in modules/interface tests --- .../interface/testsuite/test_modules_with_metaclass.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py index 14ffe0bb959..fe3e672b9b9 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py @@ -4,7 +4,9 @@ @author: pietro """ +# ruff: noqa: LOG015 from fnmatch import fnmatch +from logging import warning from grass.gunittest.case import TestCase from grass.gunittest.main import test @@ -20,6 +22,7 @@ class ModulesMeta(type): def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") print("In __new__ of ModulesMeta") def gen_test(cmd): @@ -35,6 +38,7 @@ def test(self): ] for cmd in cmds: test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) print(f"cmd is: {cmd}, test_name: {test_name}") dict[test_name] = gen_test(cmd) return type.__new__(cls, name, bases, dict) @@ -45,4 +49,5 @@ class TestModules(TestCase, metaclass=ModulesMeta): if __name__ == "__main__": + warning("Starting test of %s", __file__) test() From 7701ed7a5716945f745f27d8e085443bb52f046c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 16:07:19 +0000 Subject: [PATCH 33/53] pygrass: Stop iterating over cmds at a limit to track ModulesMeta hangs in modules/interface tests --- .../testsuite/test_modules_with_metaclass.py | 3 +- .../test_modules_with_metaclass_010.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_050.py | 54 +++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_010.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_050.py diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py index fe3e672b9b9..9ec7fcf363b 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py @@ -6,6 +6,7 @@ # ruff: noqa: LOG015 from fnmatch import fnmatch +from itertools import islice from logging import warning from grass.gunittest.case import TestCase @@ -36,7 +37,7 @@ def test(self): for c in sorted(list(get_commands()[0])) if c not in SKIP and not fnmatch(c, "g.gui.*") ] - for cmd in cmds: + for cmd in islice(cmds, 1): test_name = "test__%s" % cmd.replace(".", "_") warning("cmd is: %s, test_name: %s", cmd, test_name) print(f"cmd is: {cmd}, test_name: {test_name}") diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_010.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_010.py new file mode 100644 index 00000000000..059736793a8 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_010.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 10): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_050.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_050.py new file mode 100644 index 00000000000..3813ced193f --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_050.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 50): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() From 724d469321879d433135a8fcacd128c5cddce26d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 16:48:41 +0000 Subject: [PATCH 34/53] pygrass: Split test_modules_with_metaclass into 50 modules chunks to track ModulesMeta hangs in modules/interface tests --- ....py => test_modules_with_metaclass_100.py} | 2 +- .../test_modules_with_metaclass_150.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_200.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_250.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_300.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_350.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_400.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_450.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_500.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_550.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_600.py | 54 +++++++++++++++++++ 11 files changed, 541 insertions(+), 1 deletion(-) rename python/grass/pygrass/modules/interface/testsuite/{test_modules_with_metaclass_010.py => test_modules_with_metaclass_100.py} (96%) create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_200.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_250.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_300.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_350.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_400.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_450.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_500.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_550.py create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_600.py diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_010.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_100.py similarity index 96% rename from python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_010.py rename to python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_100.py index 059736793a8..84926da7cca 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_010.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_100.py @@ -37,7 +37,7 @@ def test(self): for c in sorted(list(get_commands()[0])) if c not in SKIP and not fnmatch(c, "g.gui.*") ] - for cmd in islice(cmds, 10): + for cmd in islice(cmds, 50, 100): test_name = "test__%s" % cmd.replace(".", "_") warning("cmd is: %s, test_name: %s", cmd, test_name) print(f"cmd is: {cmd}, test_name: {test_name}") diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py new file mode 100644 index 00000000000..4eef12a0680 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 100, 150): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_200.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_200.py new file mode 100644 index 00000000000..80cd31853b0 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_200.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 150, 200): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_250.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_250.py new file mode 100644 index 00000000000..15d70d33a2c --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_250.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 200, 250): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_300.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_300.py new file mode 100644 index 00000000000..0608429d28b --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_300.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 250, 300): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_350.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_350.py new file mode 100644 index 00000000000..ee96edd0e88 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_350.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 300, 350): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_400.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_400.py new file mode 100644 index 00000000000..69feeb78fa0 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_400.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 350, 400): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_450.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_450.py new file mode 100644 index 00000000000..17595a23567 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_450.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 400, 450): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_500.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_500.py new file mode 100644 index 00000000000..6696dddbee7 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_500.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 450, 500): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_550.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_550.py new file mode 100644 index 00000000000..a0de070191f --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_550.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 500, 550): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_600.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_600.py new file mode 100644 index 00000000000..302093256a9 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_600.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 550, None): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() From ef87db74aaf6d0cc19fafe8bb4cc98c49460d167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 17:23:12 +0000 Subject: [PATCH 35/53] pygrass: List all found cmds in test_modules_with_metaclass --- .../test_modules_with_metaclass_000.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_000.py diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_000.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_000.py new file mode 100644 index 00000000000..f6d5bce8b84 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_000.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, None): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + # dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() From c3bb88406fffe589bd40b351664e4c15f09d31a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 17:23:41 +0000 Subject: [PATCH 36/53] pygrass: Split test_modules_with_metaclass chunk between 100-150 to track ModulesMeta hangs in modules/interface tests --- .../test_modules_with_metaclass_125.py | 54 +++++++++++++++++++ .../test_modules_with_metaclass_150.py | 2 +- 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_125.py diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_125.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_125.py new file mode 100644 index 00000000000..817d7f15ee9 --- /dev/null +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_125.py @@ -0,0 +1,54 @@ +""" +Created on Tue Jun 24 09:43:53 2014 + +@author: pietro +""" + +# ruff: noqa: LOG015 +from fnmatch import fnmatch +from itertools import islice +from logging import warning + +from grass.gunittest.case import TestCase +from grass.gunittest.main import test + +from grass.script.core import get_commands +from grass.pygrass.modules.interface import Module + + +SKIP = [ + "g.parser", +] + + +class ModulesMeta(type): + def __new__(cls, name, bases, dict): + warning("In __new__ of ModulesMeta") + print("In __new__ of ModulesMeta") + + def gen_test(cmd): + def test(self): + Module(cmd) + + return test + + cmds = [ + c + for c in sorted(list(get_commands()[0])) + if c not in SKIP and not fnmatch(c, "g.gui.*") + ] + for cmd in islice(cmds, 100, 125): + test_name = "test__%s" % cmd.replace(".", "_") + warning("cmd is: %s, test_name: %s", cmd, test_name) + print(f"cmd is: {cmd}, test_name: {test_name}") + dict[test_name] = gen_test(cmd) + return type.__new__(cls, name, bases, dict) + + +class TestModules(TestCase, metaclass=ModulesMeta): + pass + + +if __name__ == "__main__": + warning("Starting test of %s", __file__) + test() diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py index 4eef12a0680..543e1189553 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py @@ -37,7 +37,7 @@ def test(self): for c in sorted(list(get_commands()[0])) if c not in SKIP and not fnmatch(c, "g.gui.*") ] - for cmd in islice(cmds, 100, 150): + for cmd in islice(cmds, 125, 150): test_name = "test__%s" % cmd.replace(".", "_") warning("cmd is: %s, test_name: %s", cmd, test_name) print(f"cmd is: {cmd}, test_name: {test_name}") From bb1be1ee6a2c6c2731331607d2fcdbb7401a2ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 18:07:28 +0000 Subject: [PATCH 37/53] gunittest: Change html_file_preview in reporters to use a context manager when opening files --- python/grass/gunittest/reporters.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/python/grass/gunittest/reporters.py b/python/grass/gunittest/reporters.py index 7c93872a656..be2ef21d483 100644 --- a/python/grass/gunittest/reporters.py +++ b/python/grass/gunittest/reporters.py @@ -9,6 +9,8 @@ :authors: Vaclav Petras """ +from __future__ import annotations + import os import datetime from pathlib import Path @@ -453,6 +455,14 @@ def wrap_stdstream_to_html(infile, outfile, module, stream): html.write(after) +def _tail( + filename: str | bytes | os.PathLike[str] | os.PathLike[bytes], n: int = 50 +) -> collections.deque[str]: + "Return the last n lines of a file" + with open(filename) as f: + return collections.deque(f, n) + + def html_file_preview(filename): before = "
"
     after = "
" @@ -469,12 +479,8 @@ def html_file_preview(filename): for line in text: html.write(color_error_line(html_escape(line))) elif size < 10 * max_size: - - def tail(filename, n): - return collections.deque(open(filename), n) # noqa: SIM115 - html.write("... (lines omitted)\n") - for line in tail(filename, 50): + for line in _tail(filename, 50): html.write(color_error_line(html_escape(line))) else: return '

Date: Sun, 24 Nov 2024 18:17:01 +0000 Subject: [PATCH 39/53] pygrass: Exclude R batch files from bin from modules/interface tests --- .../testsuite/test_modules_with_metaclass.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_050.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_075.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_100.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_125.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_150.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_200.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_250.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_300.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_350.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_400.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_450.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_500.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_550.py | 19 ++++++++++++++++++- .../test_modules_with_metaclass_600.py | 19 ++++++++++++++++++- 15 files changed, 270 insertions(+), 15 deletions(-) diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py index 9ec7fcf363b..6baef4b690a 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_050.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_050.py index 3813ced193f..c70f2fdc9a1 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_050.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_050.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_075.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_075.py index 0461f87ac03..1f30d1fc455 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_075.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_075.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_100.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_100.py index 3d91f3c5689..080cee94742 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_100.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_100.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_125.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_125.py index 817d7f15ee9..4de94327c3d 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_125.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_125.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py index 543e1189553..0d28bd8c895 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_150.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_200.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_200.py index 80cd31853b0..a46b7170cff 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_200.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_200.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_250.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_250.py index 15d70d33a2c..530b8e8026c 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_250.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_250.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_300.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_300.py index 0608429d28b..ddb8b4d632e 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_300.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_300.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_350.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_350.py index ee96edd0e88..22a8bcf89b2 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_350.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_350.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_400.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_400.py index 69feeb78fa0..ae0351957b1 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_400.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_400.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_450.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_450.py index 17595a23567..531720ded3e 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_450.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_450.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_500.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_500.py index 6696dddbee7..838e802d8e9 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_500.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_500.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_550.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_550.py index a0de070191f..3dc93a43144 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_550.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_550.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] diff --git a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_600.py b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_600.py index 302093256a9..4af640e65d1 100644 --- a/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_600.py +++ b/python/grass/pygrass/modules/interface/testsuite/test_modules_with_metaclass_600.py @@ -16,7 +16,24 @@ from grass.pygrass.modules.interface import Module -SKIP = [ +SKIP: list[str] = [ + "#Rscript2", + "ANNOUNCE", + "COPYING", + "LICENSE", + "NEWS", + "R", + "README", + "RESOURCES", + "Rpathset", + "THANKS", + "batchfiles", + "clip2r", + "copydir", + "el", + "find-miktex", + "make-batchfiles-pdf", + "movedir", "g.parser", ] From 07b4b2d793df154e525685d4da37f3156f5d3d32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 18:43:07 +0000 Subject: [PATCH 40/53] CI(OSGeo4W): Switch to windows 2022 --- .github/workflows/osgeo4w.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 6726fc5d0b2..c57f3a5adda 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: os: - - windows-2019 + - windows-2022 fail-fast: false steps: From c0598ca5d2811a643d7ca940e16f61fcb9d0b980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 18:43:59 +0000 Subject: [PATCH 41/53] CI(OSGeo4W): Setup dev drive --- .github/workflows/osgeo4w.yml | 51 ++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index c57f3a5adda..ce18926a0b3 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -33,11 +33,15 @@ jobs: git config --global core.autocrlf false git config --global core.eol lf - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: samypr100/setup-dev-drive@v3 + with: + workspace-copy: true + drive-size: 5GB - uses: msys2/setup-msys2@c52d1fa9c7492275e60fe763540fb601f5f232a1 # v2.25.0 id: msys2 with: path-type: inherit - location: D:\ + location: ${{ env.DEV_DRIVE_WORKSPACE }} update: true msystem: MINGW64 install: >- @@ -52,7 +56,8 @@ jobs: uses: echoix/setup-OSGeo4W@f4311523e39f2c8b10e34ebbc3f2ff437ecfb9ed # v0.2.0 id: osgeo4w with: - package-dir: "D:/OSGeo4W_pkg" + package-dir: "${{ env.DEV_DRIVE_WORKSPACE }}/OSGeo4W_pkg" + root: ${{ env.DEV_DRIVE_WORKSPACE }}/OSGeo4W packages: | cairo-devel freetype-devel @@ -77,9 +82,23 @@ jobs: python3-wxpython sqlite3-devel zstd-devel + - name: Get OSGEO4W_ROOT_MSYS value + id: osgeo4w-msys2 + shell: msys2 {0} + run: echo "OSGEO4W_ROOT_MSYS=$(cygpath -u $OSGEO4W_ROOT)" >> "${GITHUB_OUTPUT}" + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} + env: + OSGEO4W_ROOT: ${{ steps.osgeo4w.outputs.root }} + - name: Show OSGEO4W_ROOT_MSYS value + shell: msys2 {0} + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} + run: echo "${OSGEO4W_ROOT_MSYS}" + env: + OSGEO4W_ROOT_MSYS: ${{ steps.osgeo4w-msys2.outputs.OSGEO4W_ROOT_MSYS }} - name: Prepare ccache timestamp id: ccache_cache_timestamp shell: cmake -P {0} + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} run: | string(TIMESTAMP current_date "%Y-%m-%d-%H-%M-%S" UTC) file(APPEND "$ENV{GITHUB_OUTPUT}" "timestamp=${current_date}") @@ -87,22 +106,25 @@ jobs: uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 id: restore-cache with: - path: .ccache + path: ${{ env.DEV_DRIVE_WORKSPACE }}/.ccache key: ${{ matrix.os }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} restore-keys: | ${{ matrix.os }}-ccache- - name: Configure ccache + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} run: | echo "CCACHE_BASEDIR=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}" echo "CCACHE_DIR=${GITHUB_WORKSPACE}/.ccache" >> "${GITHUB_ENV}" echo "CCACHE_MAXSIZE=400M" >> "${GITHUB_ENV}" shell: msys2 {0} - name: Set number of cores for compilation + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} run: | echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV shell: msys2 {0} - name: Compile GRASS GIS + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} shell: msys2 {0} run: | ccache --show-stats @@ -115,22 +137,23 @@ jobs: env: PACKAGE_POSTFIX: "-dev" OSGEO4W_ROOT: ${{ steps.osgeo4w.outputs.root }} - OSGEO4W_ROOT_MSYS: "/c/OSGeo4W" + OSGEO4W_ROOT_MSYS: ${{ steps.osgeo4w-msys2.outputs.OSGEO4W_ROOT_MSYS }} PACKAGE_PATCH: ${{ github.sha }}-1 - name: Upload OSGeo4W grass package artifact if: ${{ !cancelled() }} uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 with: name: grass-osgeo4w-package-${{ matrix.os }} - path: mswindows/osgeo4w/package + path: ${{ env.DEV_DRIVE_WORKSPACE }}/mswindows/osgeo4w/package include-hidden-files: true retention-days: 3 - name: Save ccache early uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: - path: .ccache + path: ${{ env.DEV_DRIVE_WORKSPACE }}/.ccache key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Add msys to paths loaded by OSGeo4W + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} run: | $msys_path=";$env:MSYS2_ROOT/usr/bin;$env:MSYS2_ROOT/mingw64/bin" $line="set PATH=%OSGEO4W_ROOT%\bin${msys_path};%PATH%" @@ -141,6 +164,7 @@ jobs: continue-on-error: true - name: Print installed versions if: always() + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} shell: msys2 {0} run: | export ARCH=x86_64-w64-mingw32 @@ -150,20 +174,24 @@ jobs: export PYTHONHOME=${OSGEO4W_ROOT_MSYS}/apps/Python312 .github/workflows/print_versions.sh env: - OSGEO4W_ROOT_MSYS: "/c/OSGeo4W" + OSGEO4W_ROOT_MSYS: ${{ steps.osgeo4w-msys2.outputs.OSGEO4W_ROOT_MSYS }} + O4WROOT: ${{ steps.osgeo4w.outputs.root }} continue-on-error: true - name: Test executing of the grass command run: .github/workflows/test_simple.bat '${{env.O4WROOT}}\bin\grass85.bat' + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} env: O4WROOT: ${{ steps.osgeo4w.outputs.root }} - name: Test executing of the grass command in bash shell: msys2 {0} run: .github/workflows/test_simple.sh + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} continue-on-error: true # TODO - name: Test executing of the grass command in bash msys shell: msys2 {0} + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} run: | PWD="$(pwd)" SRC="${PWD}" @@ -177,12 +205,13 @@ jobs: .github/workflows/test_simple_msys.sh continue-on-error: true env: - OSGEO4W_ROOT_MSYS: "/c/OSGeo4W" + OSGEO4W_ROOT_MSYS: ${{ steps.osgeo4w-msys2.outputs.OSGEO4W_ROOT_MSYS }} OSGEO4W_ROOT: ${{ steps.osgeo4w.outputs.root }} - name: Install pytest plugins run: python -m pip install pytest-timeout - shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}"" + shell: cmd /D /E:ON /V:OFF /S /C "CALL E:/OSGeo4W/OSGeo4W.bat "{0}"" + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} - name: Run pytest with a single worker run: | call %OSGEO4W_ROOT%\apps\grass\grass85\etc\env.bat @@ -191,10 +220,12 @@ jobs: pytest --verbose --color=yes ^ --durations=0 --durations-min=0.5 ^ -ra . - shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}"" + shell: cmd /D /E:ON /V:OFF /S /C "CALL E:/OSGeo4W/OSGeo4W.bat "{0}"" + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} - name: Run tests run: .github/workflows/test_thorough.bat '${{env.O4WROOT}}\bin\grass85.bat' '${{env.O4WROOT}}\bin\python3' + working-directory: ${{ env.DEV_DRIVE_WORKSPACE }} env: O4WROOT: ${{ steps.osgeo4w.outputs.root }} From 0ed80baabf199e14130a3974428cc506cf0316b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 19:08:28 +0000 Subject: [PATCH 42/53] CI(OSGeo4W): Change locations used inside dev drive --- .github/workflows/osgeo4w.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index ce18926a0b3..03def3e0768 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -36,12 +36,12 @@ jobs: - uses: samypr100/setup-dev-drive@v3 with: workspace-copy: true - drive-size: 5GB + drive-size: 10GB - uses: msys2/setup-msys2@c52d1fa9c7492275e60fe763540fb601f5f232a1 # v2.25.0 id: msys2 with: path-type: inherit - location: ${{ env.DEV_DRIVE_WORKSPACE }} + location: ${{ env.DEV_DRIVE }} update: true msystem: MINGW64 install: >- @@ -56,8 +56,8 @@ jobs: uses: echoix/setup-OSGeo4W@f4311523e39f2c8b10e34ebbc3f2ff437ecfb9ed # v0.2.0 id: osgeo4w with: - package-dir: "${{ env.DEV_DRIVE_WORKSPACE }}/OSGeo4W_pkg" - root: ${{ env.DEV_DRIVE_WORKSPACE }}/OSGeo4W + package-dir: "${{ env.DEV_DRIVE }}/OSGeo4W_pkg" + root: ${{ env.DEV_DRIVE }}/OSGeo4W packages: | cairo-devel freetype-devel From 160c0ae01fd32da95e7c969a4f55e432f4e4cd93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:03:36 +0000 Subject: [PATCH 43/53] Test usage of vswhere --- .github/workflows/osgeo4w.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 03def3e0768..2ed2a83a29e 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -28,6 +28,21 @@ jobs: fail-fast: false steps: + - run: vswhere + continue-on-error: true + - run: | + $path = vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if ($path) { + $path = join-path $path 'Common7\Tools\vsdevcmd.bat' + if (test-path $path) { + cmd /s /c """$path"" $args && set" | where { $_ -match '(\w+)=(.*)' } | foreach { + $null = new-item -force -path "Env:\$($Matches[1])" -value $Matches[2] + } + } + } + continue-on-error: true + - run: vswhere -? + continue-on-error: true - name: Set git to use LF run: | git config --global core.autocrlf false From 28ee733c033e0bbb0b49325431ea131e31872468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:14:46 +0000 Subject: [PATCH 44/53] CI(OSGeo4W): Run vswhere to find dumpbin --- .github/workflows/osgeo4w.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 2ed2a83a29e..5951e71947f 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -43,6 +43,7 @@ jobs: continue-on-error: true - run: vswhere -? continue-on-error: true + - run: vswhere -latest -find **\dumpbin.exe - name: Set git to use LF run: | git config --global core.autocrlf false From ca0abc8070090e21e084760fff2ec2637513c8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:18:28 +0000 Subject: [PATCH 45/53] Test usage of vswhere for VC Tools --- .github/workflows/osgeo4w.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 5951e71947f..18a37286cbc 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -43,6 +43,7 @@ jobs: continue-on-error: true - run: vswhere -? continue-on-error: true + - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath - run: vswhere -latest -find **\dumpbin.exe - name: Set git to use LF run: | From 1df624802b9ea14962032a73799c81f3ba59a390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:21:19 +0000 Subject: [PATCH 46/53] Test usage of vswhere for VC Tools without a single property --- .github/workflows/osgeo4w.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 18a37286cbc..30686100d6b 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -44,6 +44,7 @@ jobs: - run: vswhere -? continue-on-error: true - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 - run: vswhere -latest -find **\dumpbin.exe - name: Set git to use LF run: | From 13d21fda7d0b78e92c26fd51fc23c6b8c913b999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:47:55 +0000 Subject: [PATCH 47/53] CI(OSGeo4W): Create a powershell function to return the path of VC Tools --- .github/workflows/osgeo4w.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 30686100d6b..a11d07918e8 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -45,6 +45,38 @@ jobs: continue-on-error: true - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 + - run: | + Function Find-VCTools + { + [CmdletBinding()] + param ( + [Parameter()] + $Arch = 'x64', + + [Parameter()] + $HostArch = 'x64', + + [Parameter(ValueFromRemainingArguments=$true)] + $Arguments + ) + + $installDir = vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if ($installDir) { + $path = join-path $installDir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt' + if (test-path $path) { + $version = gc -raw $path + if ($version) { + $version = $version.Trim() + $path = join-path $installDir "VC\Tools\MSVC\$version\bin\Host$HostArch\$Arch\" + # & $path $Arguments + return $path + } + } + } + } + Find-VCTools + Find-VCTools -Arch 'x64' + Find-VCTools -HostArch 'x64' - run: vswhere -latest -find **\dumpbin.exe - name: Set git to use LF run: | From a9106f426c5f7699e53eec0570d47b99c31e9648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:55:44 +0000 Subject: [PATCH 48/53] List env var --- .github/workflows/osgeo4w.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index a11d07918e8..632a46d5f9a 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -77,6 +77,8 @@ jobs: Find-VCTools Find-VCTools -Arch 'x64' Find-VCTools -HostArch 'x64' + - run: | + Get-ChildItem env: - run: vswhere -latest -find **\dumpbin.exe - name: Set git to use LF run: | From a4f6f3aea19916d17762312a4ac3a2a71948e64a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:57:09 +0000 Subject: [PATCH 49/53] Remove find command too long --- .github/workflows/osgeo4w.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index 632a46d5f9a..e80f4eb20de 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -79,7 +79,6 @@ jobs: Find-VCTools -HostArch 'x64' - run: | Get-ChildItem env: - - run: vswhere -latest -find **\dumpbin.exe - name: Set git to use LF run: | git config --global core.autocrlf false From 02a6d97debf0f517f8d7a7a537c1a4db94250b23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 21:06:27 +0000 Subject: [PATCH 50/53] CI(OSGeo4W): Find dumpbin directly --- .github/workflows/osgeo4w.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/osgeo4w.yml b/.github/workflows/osgeo4w.yml index e80f4eb20de..fd2a13775a1 100644 --- a/.github/workflows/osgeo4w.yml +++ b/.github/workflows/osgeo4w.yml @@ -44,6 +44,7 @@ jobs: - run: vswhere -? continue-on-error: true - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -find VC\Tools\MSVC\*\bin\Host*\*\dumpbin.exe - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 - run: | Function Find-VCTools From ebaf27aaa71924a6e7fae6914f60b8588c89a52b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 21:27:30 +0000 Subject: [PATCH 51/53] mswindows: Adapt vswhere example into bash to add the folder containing dumpbin to PATH https://github.com/microsoft/vswhere/wiki/Find-VC --- mswindows/osgeo4w/mklibs.sh | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/mswindows/osgeo4w/mklibs.sh b/mswindows/osgeo4w/mklibs.sh index e9a498ac576..5bd78810fbb 100755 --- a/mswindows/osgeo4w/mklibs.sh +++ b/mswindows/osgeo4w/mklibs.sh @@ -3,8 +3,26 @@ set -e if [ "$CI" ] ; then + HostArch="x64" + Arch="x64" + vctoolsBinPath='C:/Program Files (x86)\Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.29.30133/bin/HostX64/x64/' + echo "Calling vswhere" + vswhere -? || true + # Adapted the usage examples of vswhere for bash, and our specific needs: https://github.com/microsoft/vswhere/wiki/Find-VC + installDir="$(vswhere -latest -products "*" -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)" + echo "installDir is: $installDir" + if [ -d "$installDir" ]; then + versionFilePath="${installDir}/VC/Auxiliary/Build/Microsoft.VCToolsVersion.default.txt" + if [ -f "$versionFilePath" ]; then + versionContents=$(cat "${versionFilePath}") + echo "Default VCToolsVersion is: '$versionContents'" + vctoolsBinPath="${installDir}/VC/Tools/MSVC/${versionContents}/bin/Host${HostArch}/${Arch}/" + fi + fi + vctoolsBinPathMsys="$(cygpath -ua "$vctoolsBinPath")" # dumpbin in GH actions moved to sub-directory - export PATH="$PATH:$(cygpath -ua 'C:/Program Files (x86)\Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.29.30133/bin/HostX64/x64/')" + # export PATH="$PATH:$(cygpath -ua 'C:/Program Files (x86)\Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.29.30133/bin/HostX64/x64/')" + export PATH="$PATH:${vctoolsBinPathMsys}" # dumpbin in GH actions does not support options starting with "-" DUMPBIN_EXPORT="/EXPORTS" else From 291a2470ce911d8833e7840e3a468f6b654efdea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 22:11:34 +0000 Subject: [PATCH 52/53] Replace vswhere calls with full path since not on path --- mswindows/osgeo4w/mklibs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mswindows/osgeo4w/mklibs.sh b/mswindows/osgeo4w/mklibs.sh index 5bd78810fbb..41189e4534a 100755 --- a/mswindows/osgeo4w/mklibs.sh +++ b/mswindows/osgeo4w/mklibs.sh @@ -7,9 +7,9 @@ if [ "$CI" ] ; then Arch="x64" vctoolsBinPath='C:/Program Files (x86)\Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.29.30133/bin/HostX64/x64/' echo "Calling vswhere" - vswhere -? || true + "${ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" -? || true # Adapted the usage examples of vswhere for bash, and our specific needs: https://github.com/microsoft/vswhere/wiki/Find-VC - installDir="$(vswhere -latest -products "*" -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)" + installDir="$("${ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" -latest -products "*" -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)" echo "installDir is: $installDir" if [ -d "$installDir" ]; then versionFilePath="${installDir}/VC/Auxiliary/Build/Microsoft.VCToolsVersion.default.txt" From da04c1cc3d0a0c47c79fba1df42b672567f1b644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edouard=20Choini=C3=A8re?= <27212526+echoix@users.noreply.github.com> Date: Sun, 24 Nov 2024 22:51:29 +0000 Subject: [PATCH 53/53] mswindows: Adapt vswhere example into bash to add the folder containing dumpbin to PATH using "ProgramFiles(x86)" env var in path --- mswindows/osgeo4w/mklibs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mswindows/osgeo4w/mklibs.sh b/mswindows/osgeo4w/mklibs.sh index 41189e4534a..5cbc4c57fba 100755 --- a/mswindows/osgeo4w/mklibs.sh +++ b/mswindows/osgeo4w/mklibs.sh @@ -7,9 +7,9 @@ if [ "$CI" ] ; then Arch="x64" vctoolsBinPath='C:/Program Files (x86)\Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.29.30133/bin/HostX64/x64/' echo "Calling vswhere" - "${ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" -? || true + "$(printenv "ProgramFiles(x86)")/Microsoft Visual Studio/Installer/vswhere.exe" -? || true # Adapted the usage examples of vswhere for bash, and our specific needs: https://github.com/microsoft/vswhere/wiki/Find-VC - installDir="$("${ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" -latest -products "*" -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)" + installDir="$("$(printenv "ProgramFiles(x86)")/Microsoft Visual Studio/Installer/vswhere.exe" -latest -products "*" -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)" echo "installDir is: $installDir" if [ -d "$installDir" ]; then versionFilePath="${installDir}/VC/Auxiliary/Build/Microsoft.VCToolsVersion.default.txt"