Merge pull request #490 from ruby/katei/bundle-install-target-rbconfig

Use `bundle install --target-rbconfig` to install gems for Wasm target

Author: kateinoigakukun

.github/workflows/build.yml

@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: "3.3"
+          ruby-version: "head"
           bundler-cache: true
       - run: ./bin/setup
       - run: bundle exec rake check:type
@@ -77,7 +77,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: "3.3"
+          ruby-version: "head"
           bundler-cache: true
       - run: ./bin/setup
       - run: rake ci:pin_build_manifest
@@ -90,8 +90,8 @@ jobs:
       - name: Set matrix
         id: set-matrix
         run: |
-          rake ci:rake_task_matrix > matrix.json
-          echo "entries=$(cat matrix.json)" >> $GITHUB_OUTPUT
+          rake ci:rake_task_matrix
+          echo "entries=$(cat ci_matrix.json)" >> $GITHUB_OUTPUT
 
   rake-tasks:
     strategy:
@@ -142,21 +142,21 @@ jobs:
         if: ${{ inputs.prerel_name != '' && matrix.entry.prerelease != '' }}
       - name: rake ${{ matrix.entry.task }}
         run: ./build-exec rake --verbose ${{ matrix.entry.task }}
+      - uses: actions/upload-artifact@v4
+        if: ${{ matrix.entry.artifact }}
+        with:
+          name: ${{ matrix.entry.artifact_name }}
+          path: ${{ matrix.entry.artifact }}
       - uses: ruby/setup-ruby@v1
         if: ${{ matrix.entry.test != '' }}
         with:
-          ruby-version: "3.3"
+          ruby-version: "head"
           bundler-cache: false
       - name: rake ${{ matrix.entry.test }}
         run: |
           bundle install --with=check --without=development
           rake ${{ matrix.entry.test }}
         if: ${{ matrix.entry.test != '' }}
-      - uses: actions/upload-artifact@v4
-        if: ${{ matrix.entry.artifact }}
-        with:
-          name: ${{ matrix.entry.artifact_name }}
-          path: ${{ matrix.entry.artifact }}
 
   release-artifacts:
     needs: [rake-tasks]
@@ -176,7 +176,7 @@ jobs:
           registry-url: https://registry.npmjs.org/
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: "3.3"
+          ruby-version: "head"
           bundler-cache: true
       - run: ./bin/setup
       - run: echo "PREREL_NAME=${{ inputs.prerel_name }}" >> $GITHUB_ENV

CONTRIBUTING.md

@@ -117,6 +117,7 @@ $ gh run download <run-id>
 $ for pkg in cross-gem/pkg/ruby_wasm-*; do gem push $pkg; done
 $ gem build && gem push ruby_wasm-*.gem && rm ruby_wasm-*.gem
 $ (cd packages/gems/js/ && gem build && gem push js-*.gem && rm js-*.gem)
+$ rake bump_dev_version
 ```
 
 ## Release Channels

Gemfile

@@ -11,7 +11,8 @@ group :development do
 end
 
 group :check do
-  gem "webrick"
+  # Use the latest version of webrick for URI change in Ruby 3.4
+  gem "webrick", github: "ruby/webrick", ref: "0c600e169bd4ae267cb5eeb6197277c848323bbe"
   gem "syntax_tree", "~> 3.5"
   gem "steep"
 end

Rakefile

@@ -28,20 +28,20 @@ NPM_PACKAGES = [
   {
     name: "ruby-head-wasm-wasi",
     ruby_version: "head",
-    gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile",
+    gemfile: "packages/npm-packages/ruby-head-wasm-wasi/Gemfile",
     target: "wasm32-unknown-wasip1",
     enable_component_model: true,
   },
   {
     name: "ruby-3.3-wasm-wasi",
     ruby_version: "3.3",
-    gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile",
+    gemfile: "packages/npm-packages/ruby-3.3-wasm-wasi/Gemfile",
     target: "wasm32-unknown-wasip1"
   },
   {
     name: "ruby-3.2-wasm-wasi",
     ruby_version: "3.2",
-    gemfile: "packages/npm-packages/ruby-wasm-wasi/Gemfile",
+    gemfile: "packages/npm-packages/ruby-3.2-wasm-wasi/Gemfile",
     target: "wasm32-unknown-wasip1"
   },
   { name: "ruby-wasm-wasi", target: "wasm32-unknown-wasip1" }

bin/setup

@@ -6,7 +6,6 @@ set -vx
 root="$(cd "$(dirname "$0")/.." && pwd)"
 
 env BUNDLE_GEMFILE="$root/Gemfile" bundle install
-env BUNDLE_GEMFILE="$root/packages/npm-packages/ruby-wasm-wasi/Gemfile" bundle install
 
 # Build vendored jco if Rust toolchain is available and submodule is checked out
 if command -v rustc && [ -f vendor/jco/package.json ]; then

builders/wasm32-unknown-wasip1/Dockerfile

@@ -34,6 +34,19 @@ RUN set -eux pipefail; \
     sh -s -- -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION; \
   chmod -R a+w $RUSTUP_HOME $CARGO_HOME
 
+# Install the latest Ruby to use the latest Bundler for cross-building C extension gems.
+ADD --keep-git-dir=true https://github.com/ruby/ruby.git /buildruby
+RUN set -eux; \
+  cd /buildruby; \
+  ./autogen.sh; \
+  mkdir -p /opt/ruby; \
+  ./configure --prefix=/opt/ruby --disable-install-doc; \
+  make -j$(nproc); \
+  make install; \
+  cd /; \
+  rm -rf /buildruby
+ENV PATH=/opt/ruby/bin:$PATH
+
 ENV BUNDLE_PATH=/usr/local/gems
 RUN set -eux; \
   mkdir -p $BUNDLE_PATH; \

ext/ruby_wasm/extconf.rb

@@ -3,4 +3,7 @@
 require "mkmf"
 require "rb_sys/mkmf"
 
-create_rust_makefile("ruby_wasm/ruby_wasm")
+create_rust_makefile("ruby_wasm/ruby_wasm") do |r|
+  # We require head Ruby, so we need to fallback to compiled API
+  r.use_stable_api_compiled_fallback = true
+end

ext/ruby_wasm/src/lib.rs

@@ -252,6 +252,9 @@ impl WasiVirt {
             // Disable sockets for now since `sockets/ip-name-lookup` is not
             // supported by @bytecodealliance/preview2-shim yet
             virt.sockets(false);
+            // Disable http for now since `http` is not supported by
+            // wasmtime yet
+            virt.http(false);
             Ok(())
         })
     }

lib/ruby_wasm/build/product/crossruby.rb

@@ -71,14 +71,14 @@ def do_extconf(executor, crossruby)
         return
       end
       objdir = product_build_dir crossruby
-      rbconfig_rb = Dir.glob(File.join(crossruby.dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first
+      rbconfig_rb = crossruby.rbconfig_rb
       raise "rbconfig.rb not found" unless rbconfig_rb
       extconf_args = [
         "-C", objdir,
         "#{@srcdir}/extconf.rb",
         "--target-rbconfig=#{rbconfig_rb}",
       ]
-      extconf_args << "--enable-component-model" if @features.support_component_model?
+      extconf_args << "--disable-component-model" unless @features.support_component_model?
       executor.system crossruby.baseruby_path, *extconf_args
     end
 
@@ -111,7 +111,7 @@ def do_legacy_extconf(executor, crossruby)
         "-I#{crossruby.build_dir}",
         "--",
       ]
-      extconf_args << "--enable-component-model" if @features.support_component_model?
+      extconf_args << "--disable-component-model" unless @features.support_component_model?
       # Clear RUBYOPT to avoid loading unrelated bundle setup
       executor.system crossruby.baseruby_path,
                       *extconf_args,
@@ -301,6 +301,10 @@ def extinit_c_erb
       File.expand_path("../crossruby/extinit.c.erb", __FILE__)
     end
 
+    def rbconfig_rb
+      Dir.glob(File.join(dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first
+    end
+
     def baseruby_path
       File.join(@baseruby.install_dir, "bin/ruby")
     end

lib/ruby_wasm/cli.rb

@@ -310,7 +310,10 @@ def self.bundled_patches_path
 
     def derive_packager(options)
       __skip__ = definition = nil
-      __skip__ = if defined?(Bundler) && !options[:disable_gems]
+      features = RubyWasm::FeatureSet.derive_from_env
+      # The head ruby & dynamic linking uses "bundle" command to build gems instead of in-process integration.
+      use_in_process_gem_building = !(options[:ruby_version] == "head" && features.support_dynamic_linking?)
+      __skip__ = if defined?(Bundler) && !options[:disable_gems] && use_in_process_gem_building
         begin
           # Silence Bundler UI if --print-ruby-cache-key is specified not to bother the JSON output.
           level = options[:print_ruby_cache_key] ? :silent : Bundler.ui.level
@@ -321,17 +324,20 @@ def derive_packager(options)
           Bundler.ui.level = old_level
         end
       end
-      RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles}" if definition
+      RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles.map(&:to_s).join(", ")}" if definition
       RubyWasm::Packager.new(
         root, build_config(options), definition,
-        features: RubyWasm::FeatureSet.derive_from_env
+        features: features,
       )
     end
 
     def do_print_ruby_cache_key(packager)
       ruby_core_build = packager.ruby_core_build
       require "digest"
       digest = Digest::SHA256.new
+      # The build system key is used to invalidate the cache when the build system is updated.
+      build_system_key = 1
+      digest.update(build_system_key.to_s)
       ruby_core_build.cache_key(digest)
       hexdigest = digest.hexdigest
       require "json"

lib/ruby_wasm/packager.rb

@@ -34,7 +34,7 @@ def package(executor, dest_dir, options)
 
     ruby_core.build_gem_exts(executor, fs.bundle_dir)
 
-    fs.package_gems
+    fs.package_gems unless features.support_component_model?
     fs.remove_non_runtime_files(executor)
     if options[:stdlib]
       options[:without_stdlib_components].each do |component|

lib/ruby_wasm/packager/core.rb

@@ -1,4 +1,5 @@
 require "forwardable"
+require "pathname"
 
 class RubyWasm::Packager::Core
   def initialize(packager)
@@ -19,7 +20,6 @@ def build(executor, options)
   def build_strategy
     @build_strategy ||=
       begin
-        has_exts = @packager.specs.any? { |spec| spec.extensions.any? }
         if @packager.features.support_dynamic_linking?
           DynamicLinking.new(@packager)
         else
@@ -59,14 +59,6 @@ def specs_with_extensions
       end
     end
 
-    def wasi_exec_model
-      # TODO: Detect WASI exec-model from binary exports (_start or _initialize)
-      use_js_gem = @packager.specs.any? do |spec|
-        spec.name == "js"
-      end
-      use_js_gem ? "reactor" : "command"
-    end
-
     def with_unbundled_env(&block)
       __skip__ = if defined?(Bundler)
         Bundler.with_unbundled_env(&block)
@@ -138,12 +130,16 @@ def _link_gem_exts(executor, build, ruby_root, gem_home, module_bytes)
         wasi_sdk_path = toolchain.wasi_sdk_path
         libraries << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib)
       end
-      wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1(wasi_exec_model)
-      adapters = [wasi_adapter]
       dl_openable_libs = []
       dl_openable_libs << [File.dirname(ruby_root), Dir.glob(File.join(ruby_root, "lib", "ruby", "**", "*.so"))]
       dl_openable_libs << [gem_home, Dir.glob(File.join(gem_home, "**", "*.so"))]
 
+      has_js_so = dl_openable_libs.any? do |root, libs|
+        libs.any? { |lib| lib.end_with?("/js.so") }
+      end
+      wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1(has_js_so ? "reactor" : "command")
+      adapters = [wasi_adapter]
+
       linker = RubyWasmExt::ComponentLink.new
       linker.use_built_in_libdl(true)
       linker.stub_missing_functions(false)
@@ -187,31 +183,43 @@ def _build_gem_exts(executor, build, gem_home)
         baseruby.build(executor)
       end
 
-      exts = specs_with_extensions.flat_map do |spec, exts|
-        exts.map do |ext|
-          ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
-          ext_srcdir = File.join(spec.full_gem_path, ext_feature)
-          ext_relative_path = File.join(spec.full_name, ext_feature)
-          prod = RubyWasm::CrossRubyExtProduct.new(
-            ext_srcdir,
-            build.toolchain,
-            features: @packager.features,
-            ext_relative_path: ext_relative_path
-          )
-          [prod, spec]
-        end
-      end
+      crossruby = build.crossruby
+      rbconfig_rb = crossruby.rbconfig_rb
 
-      exts.each do |prod, spec|
-        libdir = File.join(gem_home, "gems", spec.full_name, spec.raw_require_paths.first)
-        extra_mkargs = [
-          "sitearchdir=#{libdir}",
-          "sitelibdir=#{libdir}",
-        ]
-        executor.begin_section prod.class, prod.name, "Building"
-        prod.build(executor, build.crossruby, extra_mkargs)
-        executor.end_section prod.class, prod.name
-      end
+      options = @packager.full_build_options
+      target_triplet = options[:target]
+
+      local_path = File.join("bundle", target_triplet)
+      env = {
+        "BUNDLE_APP_CONFIG" => File.join(".bundle", target_triplet),
+        "BUNDLE_PATH" => local_path,
+        "BUNDLE_WITHOUT" => "build",
+        # Do not auto-switch bundler version by Gemfile.lock
+        "BUNDLE_VERSION" => "system",
+        # FIXME: BUNDLE_PATH is set as a installation destination here, but
+        # it is also used as a source of gems to be loaded by RubyGems itself.
+        # RubyGems loads "psych" gem and if Gemfile includes "psych" gem,
+        # RubyGems tries to load "psych" gem from BUNDLE_PATH at the second
+        # time of "bundle install" command. But the extension of "psych" gem
+        # under BUNDLE_PATH is built for Wasm target, not for host platform,
+        # so it fails to load the extension.
+        #
+        # Thus we preload psych from the default LOAD_PATH here to avoid
+        # loading Wasm version of psych.so via `Kernel#require` patched by
+        # RubyGems.
+        "RUBYOPT" => "-rpsych",
+      }
+
+      args = [
+        File.join(baseruby.install_dir, "bin", "bundle"),
+        "install",
+        "--standalone",
+        "--target-rbconfig",
+        rbconfig_rb,
+      ]
+
+      executor.system(*args, env: env)
+      executor.cp_r(local_path, gem_home)
     end
 
     def cache_key(digest)
@@ -337,6 +345,14 @@ def build_gem_exts(executor, gem_home)
       # No-op because we already built extensions as part of the Ruby build
     end
 
+    def wasi_exec_model
+      # TODO: Detect WASI exec-model from binary exports (_start or _initialize)
+      use_js_gem = @packager.specs.any? do |spec|
+        spec.name == "js"
+      end
+      use_js_gem ? "reactor" : "command"
+    end
+
     def link_gem_exts(executor, ruby_root, gem_home, module_bytes)
       return module_bytes unless @packager.features.support_component_model?
 

lib/ruby_wasm/packager/file_system.rb

@@ -44,7 +44,6 @@ def remove_stdlib_component(executor, component)
     when "enc"
       # Remove all encodings except for encdb.so and transdb.so
       enc_dir = File.join(@ruby_root, "lib", "ruby", ruby_version, "wasm32-wasi", "enc")
-      puts File.join(enc_dir, "**/*.so")
       Dir.glob(File.join(enc_dir, "**/*.so")).each do |entry|
         next if entry.end_with?("encdb.so", "transdb.so")
         RubyWasm.logger.debug "Removing stdlib encoding: #{entry}"

lib/ruby_wasm/version.rb

@@ -1,3 +1,3 @@
 module RubyWasm
-  VERSION = "2.6.2"
+  VERSION = "2.6.2.dev"
 end

packages/gems/js/ext/js/extconf.rb

@@ -11,7 +11,7 @@
 
 $objs = %w[js-core.o witapi-core.o]
 
-use_component_model = enable_config("component-model", false)
+use_component_model = enable_config("component-model", true)
 $stderr.print "Building with component model: "
 $stderr.puts use_component_model ? "\e[1;32myes\e[0m" : "\e[1;31mno\e[0m"
 if use_component_model

packages/gems/js/lib/js/version.rb

@@ -1,3 +1,3 @@
 module JS
-  VERSION = "2.6.2"
+  VERSION = "2.6.2.dev"
 end

packages/npm-packages/ruby-3.2-wasm-wasi/.gitignore

@@ -1 +1,4 @@
 *.tgz
+/tmp
+/bundle
+/vendor