Merge pull request #3 from vcmi/rebuid_package

Workflow improvements

Author: IvanSavenko

.github/workflows/rebuildDependencies.yml

@@ -15,41 +15,40 @@ jobs:
         include:
           - platform: mac-intel
             os: macos-13
-            preset: macos-conan-ninja-release
+            before_install: macos.sh
             conan_profile: macos-intel
+            conan_prebuilts: dependencies-mac-intel
             conan_options: --options with_apple_system_libs=True
-            artifact_platform: intel
           - platform: mac-arm
             os: macos-13
-            preset: macos-arm-conan-ninja-release
+            before_install: macos.sh
             conan_profile: macos-arm
+            conan_prebuilts: dependencies-mac-arm
             conan_options: --options with_apple_system_libs=True
-            artifact_platform: arm
           - platform: ios
             os: macos-13
-            preset: ios-release-conan-ccache
+            before_install: macos.sh
             conan_profile: ios-arm64
+            conan_prebuilts: dependencies-ios
             conan_options: --options with_apple_system_libs=True
-          - platform: mingw
-            os: ubuntu-22.04
-            preset: windows-mingw-conan-linux
+          - platform: mingw-x86-64
+            os: ubuntu-24.04
+            before_install: mingw.sh
             conan_profile: mingw64-linux.jinja
-          - platform: mingw-32
-            os: ubuntu-22.04
-            preset: windows-mingw-conan-linux
+            conan_prebuilts: dependencies-mingw-x86-64
+          - platform: mingw-x86
+            os: ubuntu-24.04
+            before_install: mingw.sh
             conan_profile: mingw32-linux.jinja
-          - platform: android-32
-            os: macos-14
-            preset: android-conan-ninja-release
+            conan_prebuilts: dependencies-mingw-x86
+          - platform: android-armeabi-v7a
+            os: ubuntu-24.04
             conan_profile: android-32-ndk
-            conan_options: --conf tools.android:ndk_path=$ANDROID_NDK_ROOT
-            artifact_platform: armeabi-v7a
-          - platform: android-64
-            os: macos-14
-            preset: android-conan-ninja-release
+            conan_prebuilts: dependencies-android-armeabi-v7a
+          - platform: android-arm64-v8a
+            os: ubuntu-24.04
             conan_profile: android-64-ndk
-            conan_options: --conf tools.android:ndk_path=$ANDROID_NDK_ROOT
-            artifact_platform: arm64-v8a
+            conan_prebuilts: dependencies-android-arm64-v8a
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -62,21 +61,51 @@ jobs:
         repository: 'vcmi/vcmi'
         ref: 'update_prebuilts'
 
-    - name: Install dependencies
-      run: source '${{github.workspace}}/CI/${{matrix.platform}}/before_install.sh'
-      env:
-        VCMI_BUILD_PLATFORM: x64
+    - uses: actions/setup-java@v4
+      if: ${{ startsWith(matrix.platform, 'android') }}
+      with:
+        distribution: 'temurin'
+        java-version: '11'
 
-    - name: Remove old packages
-      run: rm -rf ~/.conan/data/ffmpeg ~/.conan/data/yasm ~/.conan/data/pkgconfig ~/.conan/data/xz_utils
+    - name: Prepare CI
+      if: "${{ matrix.before_install != '' }}"
+      run: source '${{github.workspace}}/CI/before_install/${{matrix.before_install}}'
 
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.10'
+    - name: Install Conan Dependencies
+      if: "${{ matrix.conan_prebuilts != '' }}"
+      run: source '${{github.workspace}}/CI/install_conan_dependencies.sh' '${{matrix.conan_prebuilts}}'
 
-    - name: Setup Conan
-      run: pip3 install 'conan<2.0'
+    - name: Remove old binary packages (non-android)
+      if: ${{ !startsWith(matrix.platform, 'android') }}
+      run: rm -rf ~/.conan/data/*/*/_/_/package
+    
+    # TODO: fix libiconv - fails to build on android (both macos and linux host)
+    - name: Remove old binary packages (android)
+      if: ${{ startsWith(matrix.platform, 'android') }}
+      run: |
+        mv ~/.conan/data/libiconv ~/
+        rm -rf ~/.conan/data/*/*/_/_/package
+        mv ~/libiconv ~/.conan/data
+
+    # Completely remove packages that were confirmed to be rebuildable using upstream recipe/sources
+    # TODO: generate entire package from scratch instead of such cleanup
+    - name: Remove old recipes
+      run: |
+        rm -rf ~/.conan/data/boost
+        rm -rf ~/.conan/data/ffmpeg
+        rm -rf ~/.conan/data/xz_utils
+        rm -rf ~/.conan/data/sdl_mixer
+        rm -rf ~/.conan/data/sdl_image
+        rm -rf ~/.conan/data/sdl_ttf
+        rm -rf ~/.conan/data/sdl
+      
+    - name: Remove old recipes (non-apple)
+      if: ${{ matrix.platform != 'ios' && matrix.platform != 'mac-intel' && matrix.platform != 'mac-arm' }}
+      run: |
+        rm -rf ~/.conan/data/sqlite3
+
+    - name: Install Conan
+      run: pipx install 'conan<2.0'
 
     - name: Generate conan profile
       run: |
@@ -94,9 +123,19 @@ jobs:
     - name: Remove builds and source code
       run: "conan remove --builds --src --force '*'"
 
-    - name: Remove Android SDK
-      if: ${{ startsWith(matrix.platform, 'android') }}
-      run: rm -rf ~/.conan/data/android-ndk
+    - name: Remove build requirements
+      run: |
+        rm -rf ~/.conan/data/android-ndk
+        rm -rf ~/.conan/data/autoconf
+        rm -rf ~/.conan/data/automake
+        rm -rf ~/.conan/data/b2
+        rm -rf ~/.conan/data/cmake
+        rm -rf ~/.conan/data/gnu-config
+        rm -rf ~/.conan/data/libtool
+        rm -rf ~/.conan/data/m4
+        rm -rf ~/.conan/data/nasm
+        rm -rf ~/.conan/data/pkgconf
+        rm -rf ~/.conan/data/yasm
 
     - name: Create dependencies archive
       run: "tar --create --xz --file dependencies-${{matrix.platform}}.txz -C ~/.conan data"

README.md

@@ -22,21 +22,13 @@ Current flow to update dependencies:
   - ffmpeg fails to find its dependencies when building with conan 1 + msvc 2019. Might be fixed in conan 2.
   - Qt fails to build due to broken string escaping in a path (conan 1 + msvc 2019)
 
-- Switch Android CI to use Linux runners instead of macOS runners (both for prebuilts and for vcmi itself)
+- Rebuild SDL_mixer and try to enable support for opus and flac. Needs investigation as to why libopus / libflac fail to build
 
-- Upgrade ubuntu runner to ubuntu-24.04 and rebuild packages using newer mingw
-
-- Rebuild boost and disable boost_url which we don't use
-
-- Rebuild SDL (including SDL_mixer and SDL_image). 
-  - Consider updating packages.
-  - Enable support for opus and flac.
-  - Remove unnecessary image formats such as gif and pcx
-  - Ensure that vcmi can load ogg/opus and flac as 'sounds' and not only as music
+- Consider removing pcx support from SDL_image
 
 - Rebuild ffmpeg with libdav1d and av1 support enabled. Needs investigation as to why dav1d fails to build on mingw and on android.
 
-- Rebuild all binaries in prebuilts package to ensure that everything is configured correctly and to replace any locally-built binaries with binaries from CI
+- Find out why libiconv fails to rebuild on Android
 
 - Rebuild entire package from scratch using latest recipes from conan, to test current version of recipes