Merge pull request #4433 from DataDog/appsec-56683-change-devise-auto-user-instrumentation-to-v3

[APPSEC-56683] Introduce automated user tracking V3

Author: Strech

.github/workflows/system-tests.yml

@@ -22,7 +22,7 @@ permissions: {}
 env:
   REGISTRY: ghcr.io
   REPO: ghcr.io/datadog/dd-trace-rb
-  SYSTEM_TESTS_REF: main # This must always be set to `main` on dd-trace-rb's master branch
+  SYSTEM_TESTS_REF: appsec-56683-rewrite-rails-devise-application
 
 jobs:
   changes:

Matrixfile

@@ -283,7 +283,7 @@
   'appsec:devise' => {
     # NOTE: JRuby bundler failed to install some dependencies https://github.com/ruby/psych/issues/700
     #       and it could be re-enabled when upstream fix the issue
-    'devise-min' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ❌ 3.0 / ❌ 3.1 / ❌ 3.2 / ❌ 3.3 / ❌ 3.4 / ❌ jruby',
+    'devise-min' => '✅ 2.5 / ✅ 2.6 / ❌ 2.7 / ❌ 3.0 / ❌ 3.1 / ❌ 3.2 / ❌ 3.3 / ❌ 3.4 / ❌ jruby',
     'devise-latest' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby'
   },
   'appsec:rails' => {

lib/datadog/appsec/anonymizer.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'digest/sha2'
+
+module Datadog
+  module AppSec
+    # Manual anonymization of the potential PII data
+    module Anonymizer
+      def self.anonymize(payload)
+        raise ArgumentError, "expected String, received #{payload.class}" unless payload.is_a?(String)
+
+        "anon_#{Digest::SHA256.hexdigest(payload)[0, 32]}"
+      end
+    end
+  end
+end

lib/datadog/appsec/configuration/settings.rb

@@ -238,10 +238,10 @@ def self.add_settings!(base)
                     Datadog.logger.warn(
                       'The appsec.auto_user_instrumentation.mode value provided is not supported. ' \
                       "Supported values are: #{AUTO_USER_INSTRUMENTATION_MODES.join(' | ')}. " \
-                      "Using default value: #{IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE}."
+                      "Using value: #{DISABLED_AUTO_USER_INSTRUMENTATION_MODE}."
                     )
 
-                    IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE
+                    DISABLED_AUTO_USER_INSTRUMENTATION_MODE
                   end
                 end
               end

lib/datadog/appsec/contrib/devise/configuration.rb

@@ -7,19 +7,11 @@ module Devise
         # A temporary configuration module to accomodate new RFC changes.
         # NOTE: DEV-3 Remove module
         module Configuration
-          MODES_CONVERSION_RULES = {
-            track_user_to_auto_instrumentation: {
-              AppSec::Configuration::Settings::SAFE_TRACK_USER_EVENTS_MODE =>
+          TRACK_USER_EVENTS_CONVERSION_RULES = {
+            AppSec::Configuration::Settings::SAFE_TRACK_USER_EVENTS_MODE =>
               AppSec::Configuration::Settings::ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE,
-              AppSec::Configuration::Settings::EXTENDED_TRACK_USER_EVENTS_MODE =>
+            AppSec::Configuration::Settings::EXTENDED_TRACK_USER_EVENTS_MODE =>
               AppSec::Configuration::Settings::IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE
-            }.freeze,
-            auto_instrumentation_to_track_user: {
-              AppSec::Configuration::Settings::ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE =>
-              AppSec::Configuration::Settings::SAFE_TRACK_USER_EVENTS_MODE,
-              AppSec::Configuration::Settings::IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE =>
-              AppSec::Configuration::Settings::EXTENDED_TRACK_USER_EVENTS_MODE
-            }.freeze
           }.freeze
 
           module_function
@@ -44,30 +36,14 @@ def auto_user_instrumentation_mode
             appsec.auto_user_instrumentation.mode
             appsec.track_user_events.mode
 
-            if !appsec.auto_user_instrumentation.options[:mode].default_precedence? &&
-                appsec.track_user_events.options[:mode].default_precedence?
-              return appsec.auto_user_instrumentation.mode
-            end
-
-            if appsec.auto_user_instrumentation.options[:mode].default_precedence?
-              return MODES_CONVERSION_RULES[:track_user_to_auto_instrumentation].fetch(
+            if !appsec.track_user_events.options[:mode].default_precedence? &&
+                appsec.auto_user_instrumentation.options[:mode].default_precedence?
+              return TRACK_USER_EVENTS_CONVERSION_RULES.fetch(
                 appsec.track_user_events.mode, appsec.auto_user_instrumentation.mode
               )
             end
 
-            identification_mode = AppSec::Configuration::Settings::IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE
-            if appsec.auto_user_instrumentation.mode == identification_mode ||
-                appsec.track_user_events.mode == AppSec::Configuration::Settings::EXTENDED_TRACK_USER_EVENTS_MODE
-              return identification_mode
-            end
-
-            AppSec::Configuration::Settings::ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE
-          end
-
-          # NOTE: Remove in next version of tracking
-          def track_user_events_mode
-            MODES_CONVERSION_RULES[:auto_instrumentation_to_track_user]
-              .fetch(auto_user_instrumentation_mode, Datadog.configuration.appsec.track_user_events.mode)
+            appsec.auto_user_instrumentation.mode
           end
         end
       end

lib/datadog/appsec/contrib/devise/data_extractor.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative '../../anonymizer'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Devise
+        # Extracts user identification data from Devise resources.
+        # Supports both regular and anonymized data extraction modes.
+        class DataExtractor
+          PRIORITY_ORDERED_ID_KEYS = [:id, 'id', :uuid, 'uuid'].freeze
+          PRIORITY_ORDERED_LOGIN_KEYS = [:email, 'email', :username, 'username', :login, 'login'].freeze
+
+          def initialize(mode:)
+            @mode = mode
+            @devise_scopes = {}
+          end
+
+          def extract_id(object)
+            return if object.nil?
+
+            if object.respond_to?(:[])
+              id = object[PRIORITY_ORDERED_ID_KEYS.find { |key| object[key] }]
+              scope = find_devise_scope(object)
+
+              id = "#{scope}:#{id}" if id && scope
+              return transform(id)
+            end
+
+            id = object.id if object.respond_to?(:id)
+            id ||= object.uuid if object.respond_to?(:uuid)
+
+            scope = find_devise_scope(object)
+            id = "#{scope}:#{id}" if id && scope
+
+            transform(id)
+          end
+
+          def extract_login(object)
+            return if object.nil?
+
+            if object.respond_to?(:[])
+              login = object[PRIORITY_ORDERED_LOGIN_KEYS.find { |key| object[key] }]
+              return transform(login)
+            end
+
+            login = object.email if object.respond_to?(:email)
+            login ||= object.username if object.respond_to?(:username)
+            login ||= object.login if object.respond_to?(:login)
+
+            transform(login)
+          end
+
+          private
+
+          def find_devise_scope(object)
+            return if ::Devise.mappings.count == 1
+
+            @devise_scopes[object.class.name] ||= begin
+              ::Devise.mappings.each_value.find { |mapping| mapping.class_name == object.class.name }&.name
+            end
+          end
+
+          def transform(value)
+            return if value.nil?
+            return value.to_s unless anonymize?
+
+            Anonymizer.anonymize(value.to_s)
+          end
+
+          def anonymize?
+            @mode == AppSec::Configuration::Settings::ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/appsec/contrib/devise/event.rb

@@ -6,6 +6,27 @@ module Contrib
       module Devise
         # Devise integration constants
         module Ext
+          EVENT_LOGIN_SUCCESS = 'users.login.success'
+          EVENT_LOGIN_FAILURE = 'users.login.failure'
+          EVENT_SIGNUP = 'users.signup'
+
+          TAG_DD_USR_ID = '_dd.appsec.usr.id'
+          TAG_DD_USR_LOGIN = '_dd.appsec.usr.login'
+          TAG_DD_SIGNUP_MODE = '_dd.appsec.events.users.signup.auto.mode'
+          TAG_DD_COLLECTION_MODE = '_dd.appsec.user.collection_mode'
+          TAG_DD_LOGIN_SUCCESS_MODE = '_dd.appsec.events.users.login.success.auto.mode'
+          TAG_DD_LOGIN_FAILURE_MODE = '_dd.appsec.events.users.login.failure.auto.mode'
+
+          TAG_USR_ID = 'usr.id'
+          TAG_SIGNUP_TRACK = 'appsec.events.users.signup.track'
+          TAG_SIGNUP_USR_ID = 'appsec.events.users.signup.usr.id'
+          TAG_SIGNUP_USR_LOGIN = 'appsec.events.users.signup.usr.login'
+          TAG_LOGIN_FAILURE_TRACK = 'appsec.events.users.login.failure.track'
+          TAG_LOGIN_FAILURE_USR_ID = 'appsec.events.users.login.failure.usr.id'
+          TAG_LOGIN_FAILURE_USR_LOGIN = 'appsec.events.users.login.failure.usr.login'
+          TAG_LOGIN_FAILURE_USR_EXISTS = 'appsec.events.users.login.failure.usr.exists'
+          TAG_LOGIN_SUCCESS_TRACK = 'appsec.events.users.login.success.track'
+          TAG_LOGIN_SUCCESS_USR_LOGIN = 'appsec.events.users.login.success.usr.login'
         end
       end
     end

lib/datadog/appsec/contrib/devise/ext.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative '../integration'
-
 require_relative 'patcher'
 
 module Datadog

lib/datadog/appsec/contrib/devise/integration.rb

@@ -1,15 +1,22 @@
 # frozen_string_literal: true
 
-require_relative 'patcher/authenticatable_patch'
-require_relative 'patcher/rememberable_patch'
-require_relative 'patcher/registration_controller_patch'
+require_relative '../../../core/utils/only_once'
+
+require_relative 'tracking_middleware'
+require_relative 'patches/signup_tracking_patch'
+require_relative 'patches/signin_tracking_patch'
+require_relative 'patches/skip_signin_tracking_patch'
 
 module Datadog
   module AppSec
     module Contrib
       module Devise
-        # Patcher for AppSec on Devise
+        # Devise patcher
         module Patcher
+          GUARD_ONCE_PER_APP = Hash.new do |hash, key|
+            hash[key] = Datadog::Core::Utils::OnlyOnce.new
+          end
+
           module_function
 
           def patched?
@@ -21,29 +28,35 @@ def target_version
           end
 
           def patch
-            patch_authenticatable_strategy
-            patch_rememberable_strategy
-            patch_registration_controller
-
-            Patcher.instance_variable_set(:@patched, true)
-          end
-
-          def patch_authenticatable_strategy
-            ::Devise::Strategies::Authenticatable.prepend(AuthenticatablePatch)
-          end
+            ::ActiveSupport.on_load(:before_initialize) do |app|
+              GUARD_ONCE_PER_APP[app].run do
+                begin
+                  app.middleware.insert_after(Warden::Manager, TrackingMiddleware)
+                rescue RuntimeError
+                  AppSec.telemetry.error('AppSec: unable to insert Devise TrackingMiddleware')
+                end
+              end
+            end
 
-          def patch_rememberable_strategy
-            return unless ::Devise::STRATEGIES.include?(:rememberable)
+            ::ActiveSupport.on_load(:after_initialize) do
+              if ::Devise::RegistrationsController.descendants.empty?
+                ::Devise::RegistrationsController.prepend(Patches::SignupTrackingPatch)
+              else
+                ::Devise::RegistrationsController.descendants.each do |controller|
+                  controller.prepend(Patches::SignupTrackingPatch)
+                end
+              end
+            end
 
-            # Rememberable strategy is required in autoloaded Rememberable model
-            ::Devise::Models::Rememberable # rubocop:disable Lint/Void
-            ::Devise::Strategies::Rememberable.prepend(RememberablePatch)
-          end
+            ::Devise::Strategies::Authenticatable.prepend(Patches::SigninTrackingPatch)
 
-          def patch_registration_controller
-            ::ActiveSupport.on_load(:after_initialize) do
-              ::Devise::RegistrationsController.prepend(RegistrationControllerPatch)
+            if ::Devise::STRATEGIES.include?(:rememberable)
+              # Rememberable strategy is required in autoloaded Rememberable model
+              require 'devise/models/rememberable'
+              ::Devise::Strategies::Rememberable.prepend(Patches::SkipSigninTrackingPatch)
             end
+
+            Patcher.instance_variable_set(:@patched, true)
           end
         end
       end