CI(OSGeo4W): package + devdrive + windows 2022

.github/workflows/osgeo4w.yml

@@ -13,7 +13,8 @@ jobs:
     name: ${{ matrix.os }} build and tests
 
     concurrency:
-      group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}-${{
+      group: >-
+        ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}-${{
         matrix.os }}
       cancel-in-progress: true
 
@@ -23,49 +24,103 @@ jobs:
     strategy:
       matrix:
         os:
-          - windows-2019
+          - windows-2022
       fail-fast: false
 
     steps:
+      - run: vswhere
+        continue-on-error: true
+      - run: |
+          $path = vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
+          if ($path) {
+              $path = join-path $path 'Common7\Tools\vsdevcmd.bat'
+              if (test-path $path) {
+                  cmd /s /c """$path"" $args && set" | where { $_ -match '(\w+)=(.*)' } | foreach {
+                      $null = new-item -force -path "Env:\$($Matches[1])" -value $Matches[2]
+                  }
+              }
+          }
+        continue-on-error: true
+      - run: vswhere -?
+        continue-on-error: true
+      - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
+      - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -find VC\Tools\MSVC\*\bin\Host*\*\dumpbin.exe
+      - run: vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64
+      - run: |
+          Function Find-VCTools
+          {
+          [CmdletBinding()]
+          param (
+            [Parameter()]
+            $Arch = 'x64',
+
+            [Parameter()]
+            $HostArch = 'x64',
+
+            [Parameter(ValueFromRemainingArguments=$true)]
+            $Arguments
+          )
+
+          $installDir = vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
+          if ($installDir) {
+            $path = join-path $installDir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt'
+            if (test-path $path) {
+              $version = gc -raw $path
+              if ($version) {
+                $version = $version.Trim()
+                $path = join-path $installDir "VC\Tools\MSVC\$version\bin\Host$HostArch\$Arch\"
+                # & $path $Arguments
+                return $path
+              }
+            }
+          }
+          }
+          Find-VCTools
+          Find-VCTools -Arch 'x64'
+          Find-VCTools -HostArch 'x64'
+      - run: |
+          Get-ChildItem env:
       - name: Set git to use LF
         run: |
           git config --global core.autocrlf false
           git config --global core.eol lf
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: samypr100/setup-dev-drive@v3
+        with:
+          workspace-copy: true
+          drive-size: 10GB
       - uses: msys2/setup-msys2@c52d1fa9c7492275e60fe763540fb601f5f232a1 # v2.25.0
+        id: msys2
         with:
           path-type: inherit
-          location: D:\
+          location: ${{ env.DEV_DRIVE }}
           update: true
           msystem: MINGW64
-          install: tar libintl make bison flex diffutils git dos2unix zip mingw-w64-x86_64-toolchain
+          install: >-
+            tar libintl make bison flex diffutils git dos2unix zip mingw-w64-x86_64-toolchain
             mingw-w64-x86_64-fftw mingw-w64-x86_64-openblas mingw-w64-x86_64-pkgconf
             mingw-w64-x86_64-gcc mingw-w64-x86_64-ccache mingw-w64-x86_64-zlib mingw-w64-x86_64-libiconv
             mingw-w64-x86_64-bzip2 mingw-w64-x86_64-gettext mingw-w64-x86_64-libsystre
-            mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-libpng
-            mingw-w64-x86_64-pcre
+            mingw-w64-x86_64-libtre-git mingw-w64-x86_64-libwinpthread-git
+            mingw-w64-x86_64-pcre mingw-w64-x86_64-omp mingw-w64-x86_64-lapack
 
       - name: Setup OSGeo4W environment
         uses: echoix/setup-OSGeo4W@f4311523e39f2c8b10e34ebbc3f2ff437ecfb9ed # v0.2.0
         id: osgeo4w
         with:
-          package-dir: "D:/OSGeo4W_pkg"
+          package-dir: "${{ env.DEV_DRIVE }}/OSGeo4W_pkg"
+          root: ${{ env.DEV_DRIVE }}/OSGeo4W
           packages: |
             cairo-devel
-            fftw
             freetype-devel
             gdal-devel
-            gdal-ecw
-            gdal-mrsid
             geos-devel
+            libjpeg-turbo-devel
             liblas-devel
             libpng-devel
             libpq-devel
             libtiff-devel
-            libxdr
             netcdf-devel
-            pdal-devel
-            pdcurses
             proj-devel
             python3-core
             python3-jupyter
@@ -75,51 +130,154 @@ jobs:
             python3-ply
             python3-pytest
             python3-pywin32
+            python3-six
             python3-wxpython
-            regex-devel
+            sqlite3-devel
             zstd-devel
-
+      - name: Get OSGEO4W_ROOT_MSYS value
+        id: osgeo4w-msys2
+        shell: msys2 {0}
+        run: echo "OSGEO4W_ROOT_MSYS=$(cygpath -u $OSGEO4W_ROOT)" >> "${GITHUB_OUTPUT}"
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
+        env:
+          OSGEO4W_ROOT: ${{ steps.osgeo4w.outputs.root }}
+      - name: Show OSGEO4W_ROOT_MSYS value
+        shell: msys2 {0}
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
+        run: echo "${OSGEO4W_ROOT_MSYS}"
+        env:
+          OSGEO4W_ROOT_MSYS: ${{ steps.osgeo4w-msys2.outputs.OSGEO4W_ROOT_MSYS }}
+      - name: Prepare ccache timestamp
+        id: ccache_cache_timestamp
+        shell: cmake -P {0}
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
+        run: |
+          string(TIMESTAMP current_date "%Y-%m-%d-%H-%M-%S" UTC)
+          file(APPEND "$ENV{GITHUB_OUTPUT}" "timestamp=${current_date}")
+      - name: Restore ccache cache files
+        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+        id: restore-cache
+        with:
+          path: ${{ env.DEV_DRIVE_WORKSPACE }}/.ccache
+          key: ${{ matrix.os }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
+          restore-keys: |
+            ${{ matrix.os }}-ccache-
+      - name: Configure ccache
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
+        run: |
+          echo "CCACHE_BASEDIR=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}"
+          echo "CCACHE_DIR=${GITHUB_WORKSPACE}/.ccache" >> "${GITHUB_ENV}"
+          echo "CCACHE_MAXSIZE=400M" >> "${GITHUB_ENV}"
+        shell: msys2 {0}
       - name: Set number of cores for compilation
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
         run: |
           echo "MAKEFLAGS=-j$(nproc)" >> $GITHUB_ENV
         shell: msys2 {0}
 
       - name: Compile GRASS GIS
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
         shell: msys2 {0}
         run: |
-          export CFLAGS="${CFLAGS} -pipe"
-          export CXXFLAGS="${CXXFLAGS} -pipe"
-          .github/workflows/build_osgeo4w.sh
-
+          ccache --show-stats
+          MSYSPATH=$(cygpath -a msys64/usr/bin):$OSGEO4W_ROOT_MSYS/bin:$(cygpath --sysdir)
+          MSYSPATH=${MSYSPATH//\/cygdrive/}
+          ccache --show-config
+          ccache --zero-stats
+          PATH=$MSYSPATH:$PATH mswindows/osgeo4w/package.sh
+          ccache --show-stats --verbose --verbose
+        env:
+          PACKAGE_POSTFIX: "-dev"
+          OSGEO4W_ROOT: ${{ steps.osgeo4w.outputs.root }}
+          OSGEO4W_ROOT_MSYS: ${{ steps.osgeo4w-msys2.outputs.OSGEO4W_ROOT_MSYS }}
+          PACKAGE_PATCH: ${{ github.sha }}-1
+      - name: Upload OSGeo4W grass package artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: grass-osgeo4w-package-${{ matrix.os }}
+          path: ${{ env.DEV_DRIVE_WORKSPACE }}/mswindows/osgeo4w/package
+          include-hidden-files: true
+          retention-days: 3
+      - name: Save ccache early
+        uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
+        with:
+          path: ${{ env.DEV_DRIVE_WORKSPACE }}/.ccache
+          key: ${{ steps.restore-cache.outputs.cache-primary-key }}
+      - name: Add msys to paths loaded by OSGeo4W
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
+        run: |
+          $msys_path=";$env:MSYS2_ROOT/usr/bin;$env:MSYS2_ROOT/mingw64/bin"
+          $line="set PATH=%OSGEO4W_ROOT%\bin${msys_path};%PATH%"
+          $line | Out-File -FilePath $env:OSGEO4W_ROOT/etc/ini/z_msys.bat
+        env:
+          MSYS2_ROOT: ${{ steps.msys2.outputs.msys2-location }}
+          OSGEO4W_ROOT: ${{ steps.osgeo4w.outputs.root }}
+          continue-on-error: true
       - name: Print installed versions
         if: always()
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
         shell: msys2 {0}
-        run: .github/workflows/print_versions.sh
+        run: |
+          export ARCH=x86_64-w64-mingw32
+          PWD="$(pwd)"
+          SRC="${PWD}"
+          export PATH="${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin:${SRC}/dist.${ARCH}/bin:${OSGEO4W_ROOT_MSYS}/apps/grass/grass85/etc:${PATH}"
+          export PYTHONHOME=${OSGEO4W_ROOT_MSYS}/apps/Python312
+          .github/workflows/print_versions.sh
+        env:
+          OSGEO4W_ROOT_MSYS: ${{ steps.osgeo4w-msys2.outputs.OSGEO4W_ROOT_MSYS }}
+          O4WROOT: ${{ steps.osgeo4w.outputs.root }}
+        continue-on-error: true
 
       - name: Test executing of the grass command
-        run: .github/workflows/test_simple.bat '${{env.O4WROOT}}\opt\grass\grass85.bat'
+        run: .github/workflows/test_simple.bat '${{env.O4WROOT}}\bin\grass85.bat'
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
         env:
           O4WROOT: ${{ steps.osgeo4w.outputs.root }}
 
       - name: Test executing of the grass command in bash
         shell: msys2 {0}
         run: .github/workflows/test_simple.sh
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
+        continue-on-error: true # TODO
+      - name: Test executing of the grass command in bash msys
+        shell: msys2 {0}
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
+        run: |
+          PWD="$(pwd)"
+          SRC="${PWD}"
+          export ARCH=x86_64-w64-mingw32
+          export PATH="${OSGEO4W_ROOT_MSYS}/bin:/usr/bin:/mingw64/bin:${SRC}/dist.${ARCH}/bin:${SRC}/dist.${ARCH}/$bash_bin:${OSGEO4W_ROOT_MSYS}/apps/grass/grass85/etc"
+          export PYTHONHOME=${OSGEO4W_ROOT_MSYS}/apps/Python312
+          export PYTHONHOME="${PYTHONHOME}"
+          export PYTHONUTF8=1
+          export GISBASE=${OSGEO4W_ROOT}/apps/grass/grass85
+          printenv | sort
+          .github/workflows/test_simple_msys.sh
+        continue-on-error: true
+        env:
+          OSGEO4W_ROOT_MSYS: ${{ steps.osgeo4w-msys2.outputs.OSGEO4W_ROOT_MSYS }}
+          OSGEO4W_ROOT: ${{ steps.osgeo4w.outputs.root }}
 
       - name: Install pytest plugins
         run: python -m pip install pytest-timeout
-        shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}""
+        shell: cmd /D /E:ON /V:OFF /S /C "CALL E:/OSGeo4W/OSGeo4W.bat "{0}""
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
       - name: Run pytest with a single worker
         run: |
-          call %OSGEO4W_ROOT%\opt\grass\etc\env.bat
+          call %OSGEO4W_ROOT%\apps\grass\grass85\etc\env.bat
           set PYTHONPATH=%GISBASE%\etc\python;%PYTHONPATH%
           path %GISBASE%\lib;%GISBASE%\bin;%PATH%
           pytest --verbose --color=yes ^
             --durations=0 --durations-min=0.5 ^
             -ra .
-        shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}""
+        shell: cmd /D /E:ON /V:OFF /S /C "CALL E:/OSGeo4W/OSGeo4W.bat "{0}""
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
 
       - name: Run tests
-        run: .github/workflows/test_thorough.bat '${{env.O4WROOT}}\opt\grass\grass85.bat' '${{env.O4WROOT}}\bin\python3'
+        run: .github/workflows/test_thorough.bat '${{env.O4WROOT}}\bin\grass85.bat' '${{env.O4WROOT}}\bin\python3'
+        working-directory: ${{ env.DEV_DRIVE_WORKSPACE }}
         env:
           O4WROOT: ${{ steps.osgeo4w.outputs.root }}
 

mswindows/osgeo4w/build_osgeo4w.sh

@@ -21,6 +21,8 @@ export C_INCLUDE_PATH=".:${OSGEO4W_ROOT_MSYS}/include:${SRC}/dist.${ARCH}/includ
 export PYTHONHOME=${OSGEO4W_ROOT_MSYS}/apps/Python312
 export ARCH=x86_64-w64-mingw32
 
+CFLAGS="$CFLAGS -pipe" \
+CXXFLAGS="$CXXFLAGS -pipe" \
 ./configure \
     --bindir=${OSGEO4W_ROOT_MSYS}/bin \
     --enable-largefile \
@@ -34,7 +36,7 @@ export ARCH=x86_64-w64-mingw32
     --with-cairo \
     --with-cairo-includes=${OSGEO4W_ROOT_MSYS}/include \
     --with-cairo-ldflags="-L${SRC}/mswindows/osgeo4w/lib -lcairo" \
-    --with-cairo-libs=$OSGEO4W_ROOT_MSYS/lib \
+    --with-cairo-libs=${OSGEO4W_ROOT_MSYS}/lib \
     --with-cxx \
     --with-fftw \
     --with-freetype \
@@ -44,6 +46,7 @@ export ARCH=x86_64-w64-mingw32
     --with-includes=${OSGEO4W_ROOT_MSYS}/include \
     --with-lapack \
     --with-liblas=${SRC}/mswindows/osgeo4w/liblas-config \
+    --with-libpng=${SRC}/mswindows/osgeo4w/libpng-config \
     --with-libs="${OSGEO4W_ROOT_MSYS}/lib ${OSGEO4W_ROOT_MSYS}/bin" \
     --with-netcdf=${OSGEO4W_ROOT_MSYS}/bin/nc-config \
     --with-nls \

mswindows/osgeo4w/env.bat.tmpl

@@ -8,6 +8,8 @@ REM Uncomment if you want to use Bash instead of Cmd
 REM Note that msys package must be also installed
 REM set GRASS_SH=%OSGEO4W_ROOT%\apps\msys\bin\sh.exe
 
+set PYTHONPATH=%OSGEO4W_ROOT%\apps\grass\grass@POSTFIX@\etc\python;%PYTHONPATH%
+set GRASS_COMPATIBILITY_TEST=0
 set GRASS_PYTHON=%OSGEO4W_ROOT%\bin\python3.exe
 set GRASS_PROJSHARE=%OSGEO4W_ROOT%\share\proj
 

mswindows/osgeo4w/libpng-config

@@ -14,7 +14,7 @@
 
 prefix="${OSGEO4W_ROOT_MSYS}"
 version="$(sed '/^#define PNG_LIBPNG_VER_STRING/!d; s/^[^"]*"\|"//g' ${prefix}/include/libpng*/png.h)"
-dll_version="$(sed '/^#define PNG_LIBPNG_VER_DLLNUM/!d; s/^[^0-9]*\|[^0-9]*$//g' ${prefix}/include/libpng*/png.h)"
+dll_version="$(sed '/^#define PNG_LIBPNG_VER_SHAREDLIB/!d; s/^[^0-9]*\|[^0-9]*$//g' ${prefix}/include/libpng*/png.h)"
 exec_prefix="${prefix}"
 libdir="${prefix}/lib"
 includedir="${prefix}/include/libpng${dll_version}"

mswindows/osgeo4w/mklibs.sh

@@ -3,8 +3,26 @@
 set -e
 
 if [ "$CI" ] ; then
+	HostArch="x64"
+	Arch="x64"
+	vctoolsBinPath='C:/Program Files (x86)\Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.29.30133/bin/HostX64/x64/'
+	echo "Calling vswhere"
+	"$(printenv "ProgramFiles(x86)")/Microsoft Visual Studio/Installer/vswhere.exe" -? || true
+	# Adapted the usage examples of vswhere for bash, and our specific needs: https://github.com/microsoft/vswhere/wiki/Find-VC
+	installDir="$("$(printenv "ProgramFiles(x86)")/Microsoft Visual Studio/Installer/vswhere.exe" -latest -products "*" -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)"
+	echo "installDir is: $installDir"
+	if [ -d "$installDir" ]; then
+		versionFilePath="${installDir}/VC/Auxiliary/Build/Microsoft.VCToolsVersion.default.txt"
+		if [ -f "$versionFilePath" ]; then
+			versionContents=$(cat "${versionFilePath}")
+			echo "Default VCToolsVersion is: '$versionContents'"
+			vctoolsBinPath="${installDir}/VC/Tools/MSVC/${versionContents}/bin/Host${HostArch}/${Arch}/"
+		fi
+	fi
+	vctoolsBinPathMsys="$(cygpath -ua "$vctoolsBinPath")"
 	# dumpbin in GH actions moved to sub-directory
-	export PATH="$PATH:$(cygpath -ua 'C:/Program Files (x86)\Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.29.30133/bin/HostX64/x64/')"
+	# export PATH="$PATH:$(cygpath -ua 'C:/Program Files (x86)\Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.29.30133/bin/HostX64/x64/')"
+	export PATH="$PATH:${vctoolsBinPathMsys}"
 	# dumpbin in GH actions does not support options starting with "-"
 	DUMPBIN_EXPORT="/EXPORTS"
 else