Move into own extension

Author: gyuheon0h

Rakefile

@@ -229,7 +229,12 @@ namespace :spec do
     t.pattern = CORE_WITH_LIBDATADOG_API.join(', ')
     t.rspec_opts = args.to_a.join(' ')
   end.tap do |t|
-    Rake::Task[t.name].enhance(["compile:libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"])
+    Rake::Task[t.name].enhance(
+      [
+        "compile:libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}",
+        "compile:datadog_runtime_stacks.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"
+      ]
+    )
   end
   # rubocop:enable Style/MultilineBlockChain
 
@@ -319,7 +324,7 @@ namespace :spec do
     rescue => e
       # Compilation failed (likely unsupported Ruby version) - tests will skip gracefully
       puts "Warning: libdatadog_api compilation failed: #{e.class}: #{e}"
-      puts "DSM tests will be skipped for this Ruby version"
+      puts 'DSM tests will be skipped for this Ruby version'
     end
 
     DSM_ENABLED_LIBRARIES.each do |task_name|
@@ -500,6 +505,10 @@ NATIVE_EXTS = [
     ext.ext_dir = 'ext/libdatadog_api'
   end,
 
+  Rake::ExtensionTask.new("datadog_runtime_stacks.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}") do |ext|
+    ext.ext_dir = 'ext/datadog_runtime_stacks'
+  end,
+
   Rake::ExtensionTask.new("datadog_profiling_native_extension.#{RUBY_VERSION}_#{RUBY_PLATFORM}") do |ext|
     ext.ext_dir = 'ext/datadog_profiling_native_extension'
   end,

datadog.gemspec

@@ -83,6 +83,7 @@ Gem::Specification.new do |spec|
 
   spec.extensions = [
     'ext/datadog_profiling_native_extension/extconf.rb',
+    'ext/datadog_runtime_stacks/extconf.rb',
     'ext/libdatadog_api/extconf.rb'
   ]
 end

ext/datadog_runtime_stacks/datadog_ruby_common.c

@@ -0,0 +1,80 @@
+#include "datadog_ruby_common.h"
+
+// IMPORTANT: Currently this file is copy-pasted between extensions. Make sure to update all versions when doing any change!
+
+void raise_unexpected_type(VALUE value, const char *value_name, const char *type_name, const char *file, int line, const char* function_name) {
+  rb_exc_raise(
+    rb_exc_new_str(
+      rb_eTypeError,
+      rb_sprintf("wrong argument %"PRIsVALUE" for '%s' (expected a %s) at %s:%d:in `%s'",
+        rb_inspect(value),
+        value_name,
+        type_name,
+        file,
+        line,
+        function_name
+      )
+    )
+  );
+}
+
+VALUE datadog_gem_version(void) {
+  VALUE ddtrace_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
+  ENFORCE_TYPE(ddtrace_module, T_MODULE);
+  VALUE version_module = rb_const_get(ddtrace_module, rb_intern("VERSION"));
+  ENFORCE_TYPE(version_module, T_MODULE);
+  VALUE version_string = rb_const_get(version_module, rb_intern("STRING"));
+  ENFORCE_TYPE(version_string, T_STRING);
+  return version_string;
+}
+
+static VALUE log_failure_to_process_tag(VALUE err_details) {
+  return log_warning(rb_sprintf("Failed to convert tag: %"PRIsVALUE, err_details));
+}
+
+__attribute__((warn_unused_result))
+ddog_Vec_Tag convert_tags(VALUE tags_as_array) {
+  ENFORCE_TYPE(tags_as_array, T_ARRAY);
+
+  long tags_count = RARRAY_LEN(tags_as_array);
+  ddog_Vec_Tag tags = ddog_Vec_Tag_new();
+
+  for (long i = 0; i < tags_count; i++) {
+    VALUE name_value_pair = rb_ary_entry(tags_as_array, i);
+
+    if (!RB_TYPE_P(name_value_pair, T_ARRAY)) {
+      ddog_Vec_Tag_drop(tags);
+      ENFORCE_TYPE(name_value_pair, T_ARRAY);
+    }
+
+    // Note: We can index the array without checking its size first because rb_ary_entry returns Qnil if out of bounds
+    VALUE tag_name = rb_ary_entry(name_value_pair, 0);
+    VALUE tag_value = rb_ary_entry(name_value_pair, 1);
+
+    if (!(RB_TYPE_P(tag_name, T_STRING) && RB_TYPE_P(tag_value, T_STRING))) {
+      ddog_Vec_Tag_drop(tags);
+      ENFORCE_TYPE(tag_name, T_STRING);
+      ENFORCE_TYPE(tag_value, T_STRING);
+    }
+
+    ddog_Vec_Tag_PushResult push_result =
+      ddog_Vec_Tag_push(&tags, char_slice_from_ruby_string(tag_name), char_slice_from_ruby_string(tag_value));
+
+    if (push_result.tag == DDOG_VEC_TAG_PUSH_RESULT_ERR) {
+      // libdatadog validates tags and may catch invalid tags that ddtrace didn't actually catch.
+      // We warn users about such tags, and then just ignore them.
+
+      int exception_state;
+      rb_protect(log_failure_to_process_tag, get_error_details_and_drop(&push_result.err), &exception_state);
+
+      // Since we are calling into Ruby code, it may raise an exception. Ensure that dynamically-allocated tags
+      // get cleaned before propagating the exception.
+      if (exception_state) {
+        ddog_Vec_Tag_drop(tags);
+        rb_jump_tag(exception_state);  // "Re-raise" exception
+      }
+    }
+  }
+
+  return tags;
+}

ext/datadog_runtime_stacks/datadog_ruby_common.h

@@ -0,0 +1,70 @@
+#pragma once
+
+// IMPORTANT: Currently this file is copy-pasted between extensions. Make sure to update all versions when doing any change!
+
+#include <ruby.h>
+#include <datadog/common.h>
+
+// Used to mark symbols to be exported to the outside of the extension.
+// Consider very carefully before tagging a function with this.
+#define DDTRACE_EXPORT __attribute__ ((visibility ("default")))
+
+// Used to mark function arguments that are deliberately left unused
+#ifdef __GNUC__
+  #define DDTRACE_UNUSED  __attribute__((unused))
+#else
+  #define DDTRACE_UNUSED
+#endif
+
+#define ADD_QUOTES_HELPER(x) #x
+#define ADD_QUOTES(x) ADD_QUOTES_HELPER(x)
+
+// Ruby has a Check_Type(value, type) that is roughly equivalent to this BUT Ruby's version is rather cryptic when it fails
+// e.g. "wrong argument type nil (expected String)". This is a replacement that prints more information to help debugging.
+#define ENFORCE_TYPE(value, type) \
+  { if (RB_UNLIKELY(!RB_TYPE_P(value, type))) raise_unexpected_type(value, ADD_QUOTES(value), ADD_QUOTES(type), __FILE__, __LINE__, __func__); }
+
+#define ENFORCE_BOOLEAN(value) \
+  { if (RB_UNLIKELY(value != Qtrue && value != Qfalse)) raise_unexpected_type(value, ADD_QUOTES(value), "true or false", __FILE__, __LINE__, __func__); }
+
+#define ENFORCE_TYPED_DATA(value, type) \
+  { if (RB_UNLIKELY(!rb_typeddata_is_kind_of(value, type))) raise_unexpected_type(value, ADD_QUOTES(value), "TypedData of type " ADD_QUOTES(type), __FILE__, __LINE__, __func__); }
+
+NORETURN(void raise_unexpected_type(VALUE value, const char *value_name, const char *type_name, const char *file, int line, const char* function_name));
+
+// Helper to retrieve Datadog::VERSION::STRING
+VALUE datadog_gem_version(void);
+
+static inline ddog_CharSlice char_slice_from_ruby_string(VALUE string) {
+  ENFORCE_TYPE(string, T_STRING);
+  ddog_CharSlice char_slice = {.ptr = RSTRING_PTR(string), .len = RSTRING_LEN(string)};
+  return char_slice;
+}
+
+static inline ddog_CharSlice char_slice_from_cstr(const char *cstr) {
+  if (cstr == NULL) {
+    return (ddog_CharSlice){.ptr = NULL, .len = 0};
+  }
+  return (ddog_CharSlice){.ptr = cstr, .len = strlen(cstr)};
+}
+
+static inline VALUE log_warning(VALUE warning) {
+  VALUE datadog_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
+  VALUE logger = rb_funcall(datadog_module, rb_intern("logger"), 0);
+
+  return rb_funcall(logger, rb_intern("warn"), 1, warning);
+}
+
+__attribute__((warn_unused_result))
+ddog_Vec_Tag convert_tags(VALUE tags_as_array);
+
+static inline VALUE ruby_string_from_error(const ddog_Error *error) {
+  ddog_CharSlice char_slice = ddog_Error_message(error);
+  return rb_str_new(char_slice.ptr, char_slice.len);
+}
+
+static inline VALUE get_error_details_and_drop(ddog_Error *error) {
+  VALUE result = ruby_string_from_error(error);
+  ddog_Error_drop(error);
+  return result;
+}

ext/datadog_runtime_stacks/extconf.rb

@@ -0,0 +1,201 @@
+# rubocop:disable Style/StderrPuts
+# rubocop:disable Style/GlobalVars
+
+require 'rubygems'
+require_relative '../libdatadog_extconf_helpers'
+
+def skip_building_extension!(reason)
+  $stderr.puts(
+    "WARN: Skipping build of datadog_runtime_stacks (#{reason}). Runtime stack capture will not be available."
+  )
+
+  fail_install_if_missing_extension = ENV['DD_FAIL_INSTALL_IF_MISSING_EXTENSION'].to_s.strip.downcase == 'true'
+
+  if fail_install_if_missing_extension
+    require 'mkmf'
+    Logging.message("[datadog] Failure cause: #{reason}")
+  else
+    File.write('Makefile', 'all install clean: # dummy makefile that does nothing')
+  end
+
+  exit
+end
+
+if ENV['DD_NO_EXTENSION'].to_s.strip.downcase == 'true'
+  skip_building_extension!('the `DD_NO_EXTENSION` environment variable is/was set to `true` during installation')
+end
+skip_building_extension!('current Ruby VM is not supported') if RUBY_ENGINE != 'ruby'
+skip_building_extension!('Microsoft Windows is not supported') if Gem.win_platform?
+
+libdatadog_issue = Datadog::LibdatadogExtconfHelpers.load_libdatadog_or_get_issue
+skip_building_extension!("issue setting up `libdatadog` gem: #{libdatadog_issue}") if libdatadog_issue
+
+require 'mkmf'
+
+# Because we can't control what compiler versions our customers use, shipping with -Werror by default is a no-go.
+# But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
+append_cflags '-Werror' if ENV['DATADOG_GEM_CI'] == 'true'
+
+# Older gcc releases may not default to C99 and we need to ask for this. This is also used:
+# * by upstream Ruby -- search for gnu99 in the codebase
+# * by msgpack, another datadog gem dependency
+#   (https://github.com/msgpack/msgpack-ruby/blob/18ce08f6d612fe973843c366ac9a0b74c4e50599/ext/msgpack/extconf.rb#L8)
+append_cflags '-std=gnu99'
+
+# Gets really noisy when we include the MJIT header, let's omit it (TODO: Use #pragma GCC diagnostic instead?)
+append_cflags "-Wno-unused-function"
+
+# Allow defining variables at any point in a function
+append_cflags '-Wno-declaration-after-statement'
+
+# If we forget to include a Ruby header, the function call may still appear to work, but then
+# cause a segfault later. Let's ensure that never happens.
+append_cflags '-Werror-implicit-function-declaration'
+
+# The native extension is not intended to expose any symbols/functions for other native libraries to use;
+# the sole exception being `Init_datadog_runtime_stacks` which needs to be visible for Ruby to call it when
+# it `dlopen`s the library.
+#
+# By setting this compiler flag, we tell it to assume that everything is private unless explicitly stated.
+# For more details see https://gcc.gnu.org/wiki/Visibility
+append_cflags '-fvisibility=hidden'
+
+# Avoid legacy C definitions
+append_cflags '-Wold-style-definition'
+
+# Enable all other compiler warnings
+append_cflags '-Wall'
+append_cflags '-Wextra'
+
+if ENV['DDTRACE_DEBUG'] == 'true'
+  $defs << '-DDD_DEBUG'
+  CONFIG['optflags'] = '-O0'
+  CONFIG['debugflags'] = '-ggdb3'
+end
+
+# If we got here, libdatadog is available and loaded
+ENV['PKG_CONFIG_PATH'] = "#{ENV["PKG_CONFIG_PATH"]}:#{Libdatadog.pkgconfig_folder}"
+Logging.message("[datadog] PKG_CONFIG_PATH set to #{ENV["PKG_CONFIG_PATH"].inspect}\n")
+$stderr.puts("Using libdatadog #{Libdatadog::VERSION} from #{Libdatadog.pkgconfig_folder}")
+
+unless pkg_config('datadog_profiling_with_rpath')
+  Logging.message("[datadog] Ruby detected the pkg-config command is #{$PKGCONFIG.inspect}\n")
+
+  if Datadog::LibdatadogExtconfHelpers.pkg_config_missing?
+    skip_building_extension!('the `pkg-config` system tool is missing')
+  else
+    skip_building_extension!('there was a problem in setting up the `libdatadog` dependency')
+  end
+end
+
+# See comments on the helper methods being used for why we need to additionally set this.
+# The extremely excessive escaping around ORIGIN below seems to be correct and was determined after a lot of
+# experimentation. We need to get these special characters across a lot of tools untouched...
+extra_relative_rpaths = [
+  Datadog::LibdatadogExtconfHelpers.libdatadog_folder_relative_to_native_lib_folder(current_folder: __dir__),
+  *Datadog::LibdatadogExtconfHelpers.libdatadog_folder_relative_to_ruby_extensions_folders,
+]
+extra_relative_rpaths.each { |folder| $LDFLAGS += " -Wl,-rpath,$$$\\\\{ORIGIN\\}/#{folder.to_str}" }
+Logging.message("[datadog] After pkg-config $LDFLAGS were set to: #{$LDFLAGS.inspect}\n")
+
+# Enable access to Ruby VM internal headers for runtime stack walking
+# Ruby version compatibility definitions
+
+# On Ruby 3.5, we can't ask the object_id from IMEMOs (https://github.com/ruby/ruby/pull/13347)
+$defs << "-DNO_IMEMO_OBJECT_ID" unless RUBY_VERSION < "3.5"
+
+# On Ruby 2.5 and 3.3, this symbol was not visible. It is on 2.6 to 3.2, as well as 3.4+
+$defs << "-DNO_RB_OBJ_INFO" if RUBY_VERSION.start_with?("2.5", "3.3")
+
+# On older Rubies, M:N threads were not available
+$defs << "-DNO_MN_THREADS_AVAILABLE" if RUBY_VERSION < "3.3"
+
+# On older Rubies, we did not need to include the ractor header (this was built into the MJIT header)
+$defs << "-DNO_RACTOR_HEADER_INCLUDE" if RUBY_VERSION < "3.3"
+
+# On older Rubies, some of the Ractor internal APIs were directly accessible
+$defs << "-DUSE_RACTOR_INTERNAL_APIS_DIRECTLY" if RUBY_VERSION < "3.3"
+
+# On older Rubies, the first_lineno inside a location was a VALUE and not a int (https://github.com/ruby/ruby/pull/6430)
+$defs << "-DNO_INT_FIRST_LINENO" if RUBY_VERSION < "3.2"
+
+# On older Rubies, there are no Ractors
+$defs << "-DNO_RACTORS" if RUBY_VERSION < "3"
+
+# On older Rubies, rb_imemo_name did not exist
+$defs << "-DNO_IMEMO_NAME" if RUBY_VERSION < "3"
+
+# Tag the native extension library with the Ruby version and Ruby platform.
+# This makes it easier for development (avoids "oops I forgot to rebuild when I switched my Ruby") and ensures that
+# the wrong library is never loaded.
+# When requiring, we need to use the exact same string, including the version and the platform.
+EXTENSION_NAME = "datadog_runtime_stacks.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}".freeze
+
+CAN_USE_MJIT_HEADER = RUBY_VERSION.start_with?("2.6", "2.7", "3.0.", "3.1.", "3.2.")
+
+mjit_header_worked = false
+
+if CAN_USE_MJIT_HEADER
+  mjit_header_file_name = "rb_mjit_min_header-#{RUBY_VERSION}.h"
+
+  # Validate that the mjit header can actually be compiled on this system.
+  original_common_headers = MakeMakefile::COMMON_HEADERS
+  MakeMakefile::COMMON_HEADERS = "".freeze
+  if have_macro("RUBY_MJIT_H", mjit_header_file_name)
+    MakeMakefile::COMMON_HEADERS = original_common_headers
+
+    $defs << "-DRUBY_MJIT_HEADER='\"#{mjit_header_file_name}\"'"
+
+    # NOTE: This needs to come after all changes to $defs
+    create_header
+
+    # Note: -Wunused-parameter flag is intentionally added here only after MJIT header validation
+    append_cflags "-Wunused-parameter"
+
+    create_makefile(EXTENSION_NAME)
+    mjit_header_worked = true
+  else
+    # MJIT header compilation failed, fallback to datadog-ruby_core_source
+    $stderr.puts "MJIT header compilation failed, falling back to datadog-ruby_core_source"
+    MakeMakefile::COMMON_HEADERS = original_common_headers
+  end
+end
+
+# The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on
+# the datadog-ruby_core_source gem to get access to private VM headers.
+# We also use it as a fallback when MJIT header compilation fails.
+unless mjit_header_worked
+  create_header
+
+  require "datadog/ruby_core_source"
+  dir_config("ruby") # allow user to pass in non-standard core include directory
+
+  # Workaround for mkmf issue with $CPPFLAGS
+  Datadog::RubyCoreSource.define_singleton_method(:with_cppflags) do |newflags, &block|
+    super("#{newflags} #{$CPPFLAGS}", &block)
+  end
+
+  makefile_created = Datadog::RubyCoreSource
+    .create_makefile_with_core(
+      proc do
+        headers_available =
+          have_header("vm_core.h") &&
+          have_header("iseq.h") &&
+          (RUBY_VERSION < "3.3" || have_header("ractor_core.h"))
+
+        if headers_available
+          append_cflags "-Wunused-parameter"
+        end
+
+        headers_available
+      end,
+      EXTENSION_NAME
+    )
+
+  unless makefile_created
+    skip_building_extension!('required Ruby VM internal headers are not available for this Ruby version')
+  end
+end
+
+# rubocop:enable Style/GlobalVars
+# rubocop:enable Style/StderrPuts