diff --git a/browser/ui/webui/ai_chat/ai_chat_ui_page_handler.cc b/browser/ui/webui/ai_chat/ai_chat_ui_page_handler.cc index bcafe355b757..39704b8a6fce 100644 --- a/browser/ui/webui/ai_chat/ai_chat_ui_page_handler.cc +++ b/browser/ui/webui/ai_chat/ai_chat_ui_page_handler.cc @@ -14,6 +14,8 @@ #include "base/functional/callback_forward.h" #include "brave/browser/ai_chat/ai_chat_service_factory.h" #include "brave/browser/ai_chat/ai_chat_urls.h" +#include "brave/browser/brave_browser_process.h" +#include "brave/browser/misc_metrics/process_misc_metrics.h" #include "brave/browser/ui/side_panel/ai_chat/ai_chat_side_panel_utils.h" #include "brave/components/ai_chat/core/browser/ai_chat_service.h" #include "brave/components/ai_chat/core/browser/constants.h" @@ -137,6 +139,8 @@ AIChatUIPageHandler::AIChatUIPageHandler( mojo::PendingReceiver receiver) : owner_web_contents_(owner_web_contents), profile_(profile), + ai_chat_metrics_( + g_brave_browser_process->process_misc_metrics()->ai_chat_metrics()), receiver_(this, std::move(receiver)) { // Standalone mode means Chat is opened as its own tab in the tab strip and // not a side panel. chat_context_web_contents is nullptr in that case @@ -144,6 +148,11 @@ AIChatUIPageHandler::AIChatUIPageHandler( profile_, ServiceAccessType::EXPLICIT_ACCESS); const bool is_standalone = chat_context_web_contents == nullptr; if (!is_standalone) { +#if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) + if (ai_chat_metrics_) { + ai_chat_metrics_->RecordSidebarUsage(); + } +#endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) active_chat_tab_helper_ = ai_chat::AIChatTabHelper::FromWebContents(chat_context_web_contents); chat_tab_helper_observation_.Observe(active_chat_tab_helper_); @@ -190,6 +199,11 @@ void AIChatUIPageHandler::OpenConversationFullPage( const std::string& conversation_uuid) { CHECK(ai_chat::features::IsAIChatHistoryEnabled()); CHECK(active_chat_tab_helper_); +#if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) + if (ai_chat_metrics_) { + ai_chat_metrics_->RecordFullPageSwitch(); + } +#endif active_chat_tab_helper_->web_contents()->OpenURL( { ConversationUrl(conversation_uuid), diff --git a/browser/ui/webui/ai_chat/ai_chat_ui_page_handler.h b/browser/ui/webui/ai_chat/ai_chat_ui_page_handler.h index 7c5676ba8cf7..0c6758754454 100644 --- a/browser/ui/webui/ai_chat/ai_chat_ui_page_handler.h +++ b/browser/ui/webui/ai_chat/ai_chat_ui_page_handler.h @@ -100,6 +100,7 @@ class AIChatUIPageHandler : public mojom::AIChatUIHandler, raw_ptr owner_web_contents_ = nullptr; raw_ptr favicon_service_ = nullptr; raw_ptr profile_ = nullptr; + raw_ptr ai_chat_metrics_; base::CancelableTaskTracker favicon_task_tracker_; diff --git a/chromium_src/chrome/browser/autocomplete/chrome_autocomplete_provider_client.cc b/chromium_src/chrome/browser/autocomplete/chrome_autocomplete_provider_client.cc index 4c4f73142c23..86167d9283ec 100644 --- a/chromium_src/chrome/browser/autocomplete/chrome_autocomplete_provider_client.cc +++ b/chromium_src/chrome/browser/autocomplete/chrome_autocomplete_provider_client.cc @@ -110,12 +110,12 @@ void ChromeAutocompleteProviderClient::OpenLeo(const std::u16string& query) { base::Time::Now(), std::nullopt /* edits */, false /* from_brave_search_SERP */); - conversation_handler->SubmitHumanConversationEntry(std::move(turn)); - ai_chat::AIChatMetrics* metrics = g_brave_browser_process->process_misc_metrics()->ai_chat_metrics(); CHECK(metrics); metrics->RecordOmniboxOpen(); + + conversation_handler->SubmitHumanConversationEntry(std::move(turn)); #endif } diff --git a/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.cc b/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.cc index 13a41d20f20d..2f66cad3992a 100644 --- a/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.cc +++ b/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.cc @@ -544,6 +544,12 @@ void BraveRenderViewContextMenu::ExecuteAIChatCommand(int command) { auto [action_type, p3a_action] = GetActionTypeAndP3A(command); auto selected_text = base::UTF16ToUTF8(params_.selection_text); + auto* ai_chat_metrics = + g_brave_browser_process->process_misc_metrics()->ai_chat_metrics(); + if (ai_chat_metrics) { + ai_chat_metrics->OnQuickActionStatusChange(true); + } + if (rewrite_in_place) { source_web_contents_->SetUserData(kAIChatRewriteDataKey, std::make_unique()); @@ -594,9 +600,9 @@ void BraveRenderViewContextMenu::ExecuteAIChatCommand(int command) { conversation->SubmitSelectedText(selected_text, action_type); } - g_brave_browser_process->process_misc_metrics() - ->ai_chat_metrics() - ->RecordContextMenuUsage(p3a_action); + if (ai_chat_metrics) { + ai_chat_metrics->RecordContextMenuUsage(p3a_action); + } } void BraveRenderViewContextMenu::BuildAIChatMenu() { diff --git a/components/ai_chat/core/browser/ai_chat_metrics.cc b/components/ai_chat/core/browser/ai_chat_metrics.cc index c1635d078e03..36a6109038d6 100644 --- a/components/ai_chat/core/browser/ai_chat_metrics.cc +++ b/components/ai_chat/core/browser/ai_chat_metrics.cc @@ -8,25 +8,20 @@ #include #include -#include #include -#include #include -#include #include #include -#include #include #include "base/check.h" #include "base/containers/fixed_flat_map.h" #include "base/functional/bind.h" #include "base/location.h" -#include "base/metrics/histogram_base.h" #include "base/metrics/histogram_functions_internal_overloads.h" #include "base/metrics/histogram_macros.h" -#include "base/numerics/clamped_math.h" #include "base/time/time.h" +#include "brave/components/ai_chat/core/common/mojom/ai_chat.mojom-shared.h" #include "brave/components/ai_chat/core/common/pref_names.h" #include "brave/components/p3a_utils/bucket.h" #include "brave/components/p3a_utils/feature_usage.h" @@ -42,12 +37,21 @@ using sidebar::features::SidebarDefaultMode; constexpr base::TimeDelta kReportInterval = base::Hours(24); constexpr base::TimeDelta kReportDebounceDelay = base::Seconds(3); +constexpr base::TimeDelta kFirstChatPromptsReportDebounceDelay = + base::Minutes(10); + +constexpr int kFirstChatPromptsBuckets[] = {1, 3, 6, 10}; constexpr int kChatCountBuckets[] = {1, 5, 10, 20, 50}; constexpr int kAvgPromptCountBuckets[] = {2, 5, 10, 20}; +constexpr int kChatHistoryUsageBuckets[] = {0, 1, 4, 10, 25, 50, 75}; +constexpr int kMaxChatDurationBuckets[] = {1, 2, 5, 15, 30, 60}; +constexpr int kRateLimitsBuckets[] = {0, 1, 3, 5}; +constexpr int kContextLimitsBuckets[] = {0, 2, 5, 10}; constexpr base::TimeDelta kPremiumCheckInterval = base::Days(1); #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) +constexpr int kFullPageSwitchesBuckets[] = {0, 5, 25, 50}; // Value -1 is added to buckets to add padding for the "less than 1% option" constexpr int kOmniboxOpenBuckets[] = {-1, 0, 3, 5, 10, 25}; constexpr int kContextMenuUsageBuckets[] = {0, 1, 2, 5, 10, 20, 50}; @@ -92,6 +96,24 @@ constexpr auto kEntryPointKeys = #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) +constexpr char kOmniboxInputKey[] = "omnibox_input"; +constexpr char kConversationStarterKey[] = "conversation_starter"; +constexpr char kPageSummaryKey[] = "page_summary"; +constexpr char kTextInputWithPageKey[] = "text_input_with_page"; +constexpr char kTextInputWithoutPageKey[] = "text_input_without_page"; +constexpr char kTextInputViaFullPageKey[] = "text_input_via_full_page"; +constexpr char kQuickActionKey[] = "quick_action"; + +constexpr auto kContextSourceKeys = + base::MakeFixedFlatMap( + {{ContextSource::kOmniboxInput, kOmniboxInputKey}, + {ContextSource::kConversationStarter, kConversationStarterKey}, + {ContextSource::kPageSummary, kPageSummaryKey}, + {ContextSource::kTextInputWithPage, kTextInputWithPageKey}, + {ContextSource::kTextInputWithoutPage, kTextInputWithoutPageKey}, + {ContextSource::kTextInputViaFullPage, kTextInputViaFullPageKey}, + {ContextSource::kQuickAction, kQuickActionKey}}); + void ReportHistogramForSidebarExperiment( int value, base::fixed_flat_map name_map) { @@ -109,6 +131,31 @@ void ReportHistogramForSidebarExperiment( } } +template +uint64_t ReportMostUsedMetric( + const base::flat_map>& storages, + const char* histogram_name, + EnumType max_value) { + uint64_t total = 0; + uint64_t total_max = 0; + std::optional most_used; + + for (int i = 0; i <= static_cast(max_value); i++) { + EnumType enum_value = static_cast(i); + uint64_t weekly_total = storages.at(enum_value)->GetWeeklySum(); + if (weekly_total > total_max) { + most_used = enum_value; + total_max = weekly_total; + } + total += weekly_total; + } + + if (most_used) { + UMA_HISTOGRAM_ENUMERATION(histogram_name, *most_used); + } + return total; +} + constexpr auto kEnabledHistogramNames = base::MakeFixedFlatMap( {{SidebarDefaultMode::kOff, kEnabledHistogramName}, @@ -139,8 +186,15 @@ AIChatMetrics::AIChatMetrics(PrefService* local_state) local_state->GetBoolean(prefs::kBraveChatP3ALastPremiumStatus)), chat_count_storage_(local_state, prefs::kBraveChatP3AChatCountWeeklyStorage), + chat_with_history_count_storage_( + local_state, + prefs::kBraveChatP3AChatWithHistoryCountWeeklyStorage), + chat_durations_storage_(local_state, + prefs::kBraveChatP3AChatDurationsWeeklyStorage), prompt_count_storage_(local_state, prefs::kBraveChatP3APromptCountWeeklyStorage), + rate_limit_storage_(local_state, prefs::kBraveChatP3ARateLimitStops), + context_limit_storage_(local_state, prefs::kBraveChatP3AContextLimits), #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) omnibox_open_storage_(local_state, prefs::kBraveChatP3AOmniboxOpenWeeklyStorage, @@ -149,6 +203,9 @@ AIChatMetrics::AIChatMetrics(PrefService* local_state) local_state, prefs::kBraveChatP3AOmniboxAutocompleteWeeklyStorage, 14), + sidebar_usage_storage_(local_state, prefs::kBraveChatP3ASidebarUsages), + full_page_switch_storage_(local_state, + prefs::kBraveChatP3AFullPageSwitches), #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) local_state_(local_state) { #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) @@ -161,16 +218,25 @@ AIChatMetrics::AIChatMetrics(PrefService* local_state) for (int i = 0; i <= static_cast(EntryPoint::kMaxValue); i++) { EntryPoint entry_point = static_cast(i); entry_point_storages_[entry_point] = std::make_unique( - local_state_, prefs::kBraveChatP3AContextMenuUsages, + local_state_, prefs::kBraveChatP3AEntryPointUsages, kEntryPointKeys.at(entry_point)); } #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) + for (int i = 0; i <= static_cast(ContextSource::kMaxValue); i++) { + ContextSource context_source = static_cast(i); + context_source_storages_[context_source] = std::make_unique( + local_state_, prefs::kBraveChatP3AContextSourceUsages, + kContextSourceKeys.at(context_source)); + } } AIChatMetrics::~AIChatMetrics() = default; void AIChatMetrics::RegisterPrefs(PrefRegistrySimple* registry) { registry->RegisterListPref(prefs::kBraveChatP3AChatCountWeeklyStorage); + registry->RegisterListPref( + prefs::kBraveChatP3AChatWithHistoryCountWeeklyStorage); + registry->RegisterListPref(prefs::kBraveChatP3AChatDurationsWeeklyStorage); registry->RegisterListPref(prefs::kBraveChatP3APromptCountWeeklyStorage); #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) registry->RegisterListPref(prefs::kBraveChatP3AOmniboxOpenWeeklyStorage); @@ -184,6 +250,18 @@ void AIChatMetrics::RegisterPrefs(PrefRegistrySimple* registry) { registry->RegisterTimePref(prefs::kBraveChatP3AFirstUsageTime, {}); registry->RegisterTimePref(prefs::kBraveChatP3ALastUsageTime, {}); registry->RegisterBooleanPref(prefs::kBraveChatP3AUsedSecondDay, false); + registry->RegisterBooleanPref(prefs::kBraveChatP3AFirstChatPromptsReported, + false); + registry->RegisterDictionaryPref(prefs::kBraveChatP3AContextSourceUsages); + registry->RegisterDictionaryPref(prefs::kBraveChatP3AEntryPointUsages); + registry->RegisterListPref(prefs::kBraveChatP3ASidebarUsages); + registry->RegisterListPref(prefs::kBraveChatP3AFullPageSwitches); + registry->RegisterListPref(prefs::kBraveChatP3ARateLimitStops); + registry->RegisterListPref(prefs::kBraveChatP3AContextLimits); +} + +void AIChatMetrics::Bind(mojo::PendingReceiver receiver) { + receivers_.Add(this, std::move(receiver)); } void AIChatMetrics::RecordEnabled( @@ -253,11 +331,21 @@ void AIChatMetrics::OnPremiumStatusUpdated(bool is_enabled, RecordEnabled(is_enabled, is_new_user, {}); } -void AIChatMetrics::RecordNewChat() { - chat_count_storage_.AddDelta(1); -} +void AIChatMetrics::RecordNewPrompt(ConversationHandlerForMetrics* handler, + mojom::ConversationPtr& conversation, + mojom::ConversationTurnPtr& entry) { + const auto& uuid = conversation->uuid; + if (!conversation_start_times_.contains(uuid)) { + chat_count_storage_.AddDelta(1); + if (handler->GetConversationHistorySize() > 1) { + chat_with_history_count_storage_.AddDelta(1); + } + conversation_start_times_[uuid] = base::Time::Now(); + } + + chat_durations_storage_.ReplaceTodaysValueIfGreater( + (base::Time::Now() - conversation_start_times_[uuid]).InMinutes()); -void AIChatMetrics::RecordNewPrompt() { ReportHistogramForSidebarExperiment(is_premium_ ? 2 : 1, kUsageDailyHistogramNames); ReportHistogramForSidebarExperiment(is_premium_ ? 2 : 1, @@ -272,10 +360,75 @@ void AIChatMetrics::RecordNewPrompt() { report_debounce_timer_.Start( FROM_HERE, kReportDebounceDelay, base::BindOnce(&AIChatMetrics::ReportChatCounts, base::Unretained(this))); + MaybeReportFirstChatPrompts(true); + + RecordContextSource(handler, entry); + +#if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) + ReportFullPageUsageMetric(); +#endif + + if (handler->should_send_page_contents() && + conversation->associated_content && + conversation->associated_content->content_used_percentage < 100) { + context_limit_storage_.AddDelta(1u); + } + ReportLimitMetrics(); +} + +void AIChatMetrics::MaybeRecordLastError( + ConversationHandlerForMetrics* handler) { + if (handler->current_error() == mojom::APIError::RateLimitReached && + (base::Time::Now() - + local_state_->GetTime(prefs::kBraveChatP3AFirstUsageTime)) <= + base::Days(7)) { + rate_limit_storage_.AddDelta(1u); + ReportLimitMetrics(); + } +} + +void AIChatMetrics::RecordConversationUnload( + std::string_view conversation_uuid) { + conversation_start_times_.erase(conversation_uuid); + MaybeReportFirstChatPrompts(false); +} + +void AIChatMetrics::RecordConversationsCleared() { + conversation_start_times_.clear(); + MaybeReportFirstChatPrompts(false); +} + +void AIChatMetrics::OnSendingPromptWithFullPage() { + prompted_via_full_page_ = true; +} + +void AIChatMetrics::OnQuickActionStatusChange(bool is_enabled) { + prompted_via_quick_action_ = is_enabled; +} + +void AIChatMetrics::MaybeReportFirstChatPrompts(bool new_prompt_made) { + if (local_state_->GetBoolean(prefs::kBraveChatP3AFirstChatPromptsReported)) { + return; + } + if (new_prompt_made) { + first_chat_report_debounce_timer_.Start( + FROM_HERE, kFirstChatPromptsReportDebounceDelay, + base::BindOnce(&AIChatMetrics::MaybeReportFirstChatPrompts, + base::Unretained(this), false)); + return; + } + auto prompt_count = prompt_count_storage_.GetWeeklySum(); + if (prompt_count == 0) { + return; + } + p3a_utils::RecordToHistogramBucket(kFirstChatPromptsHistogramName, + kFirstChatPromptsBuckets, prompt_count); + local_state_->SetBoolean(prefs::kBraveChatP3AFirstChatPromptsReported, true); } #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) void AIChatMetrics::RecordOmniboxOpen() { + prompted_via_omnibox_ = true; HandleOpenViaEntryPoint(EntryPoint::kOmniboxItem); omnibox_open_storage_.AddDelta(1); omnibox_autocomplete_storage_.AddDelta(1); @@ -304,6 +457,17 @@ void AIChatMetrics::HandleOpenViaEntryPoint(EntryPoint entry_point) { ReportEntryPointUsageMetric(); } + +void AIChatMetrics::RecordSidebarUsage() { + sidebar_usage_storage_.AddDelta(1u); + ReportFullPageUsageMetric(); +} + +void AIChatMetrics::RecordFullPageSwitch() { + full_page_switch_storage_.AddDelta(1u); + ReportFullPageUsageMetric(); +} + #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) void AIChatMetrics::ReportAllMetrics() { @@ -312,10 +476,13 @@ void AIChatMetrics::ReportAllMetrics() { base::BindOnce(&AIChatMetrics::ReportAllMetrics, base::Unretained(this))); ReportChatCounts(); ReportFeatureUsageMetrics(); + ReportContextSource(); + ReportLimitMetrics(); #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) ReportOmniboxCounts(); ReportContextMenuMetrics(); ReportEntryPointUsageMetric(); + ReportFullPageUsageMetric(); #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) } @@ -351,6 +518,81 @@ void AIChatMetrics::ReportChatCounts() { // TODO(djandries): remove the following report when Nebula experiment is over p3a_utils::RecordToHistogramBucket(kChatCountNebulaHistogramName, kChatCountBuckets, chat_count); + + uint64_t max_chat_duration = + chat_durations_storage_.GetHighestValueInPeriod(); + p3a_utils::RecordToHistogramBucket(kMaxChatDurationHistogramName, + kMaxChatDurationBuckets, + max_chat_duration); + + uint64_t chat_with_history_count = + chat_with_history_count_storage_.GetWeeklySum(); + int history_percentage = static_cast(std::ceil( + chat_count > 0 + ? (static_cast(chat_with_history_count) / chat_count) * 100.0 + : 0.0)); + + p3a_utils::RecordToHistogramBucket(kChatHistoryUsageHistogramName, + kChatHistoryUsageBuckets, + history_percentage); +} + +void AIChatMetrics::RecordContextSource(ConversationHandlerForMetrics* handler, + mojom::ConversationTurnPtr& entry) { + ContextSource context = ContextSource::kTextInputWithoutPage; + if (prompted_via_omnibox_) { + context = ContextSource::kOmniboxInput; + } else if (entry->action_type == mojom::ActionType::SUMMARIZE_PAGE || + entry->action_type == mojom::ActionType::SUMMARIZE_VIDEO) { + context = ContextSource::kPageSummary; + } else if (handler->GetConversationHistorySize() == 1 && + entry->action_type == mojom::ActionType::CONVERSATION_STARTER) { + UMA_HISTOGRAM_BOOLEAN(kUsedConversationStarterHistogramName, true); + context = ContextSource::kConversationStarter; + } else if (prompted_via_quick_action_) { + context = ContextSource::kQuickAction; + } else if (prompted_via_full_page_) { + context = ContextSource::kTextInputViaFullPage; + } else if (handler->should_send_page_contents()) { + context = ContextSource::kTextInputWithPage; + } + prompted_via_omnibox_ = false; + prompted_via_full_page_ = false; + prompted_via_quick_action_ = false; + + context_source_storages_.at(context)->AddDelta(1u); + ReportContextSource(); +} + +void AIChatMetrics::ReportContextSource() { + if (!is_enabled_) { + return; + } + if (chat_count_storage_.GetWeeklySum() == 0) { + // Do not report if AI chat was not used in the past week. + return; + } + + ReportMostUsedMetric(context_source_storages_, + kMostUsedContextSourceHistogramName, + ContextSource::kMaxValue); +} + +void AIChatMetrics::ReportLimitMetrics() { + if (!is_enabled_) { + return; + } + if (chat_count_storage_.GetWeeklySum() == 0) { + // Do not report if AI chat was not used in the past week. + return; + } + + p3a_utils::RecordToHistogramBucket(kRateLimitStopsHistogramName, + kRateLimitsBuckets, + rate_limit_storage_.GetWeeklySum()); + p3a_utils::RecordToHistogramBucket(kContextLimitsHistogramName, + kContextLimitsBuckets, + context_limit_storage_.GetWeeklySum()); } #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) @@ -394,31 +636,18 @@ void AIChatMetrics::ReportOmniboxCounts() { } void AIChatMetrics::ReportContextMenuMetrics() { - uint64_t total_usages = 0; - uint64_t action_total_max = 0; - std::optional most_used_action; - - for (int i = static_cast(ContextMenuAction::kSummarize); - i <= static_cast(ContextMenuAction::kMaxValue); i++) { - ContextMenuAction action = static_cast(i); - uint64_t action_total = - context_menu_usage_storages_[action]->GetWeeklySum(); - total_usages += action_total; - if (action_total > action_total_max) { - most_used_action = action; - action_total_max = action_total; - } + if (!is_enabled_) { + return; } + auto total_usages = ReportMostUsedMetric( + context_menu_usage_storages_, kMostUsedContextMenuActionHistogramName, + ContextMenuAction::kMaxValue); + p3a_utils::RecordFeatureLastUsageTimeMetric( local_state_, prefs::kBraveChatP3ALastContextMenuUsageTime, kContextMenuLastUsageTimeHistogramName, true); - if (most_used_action && is_enabled_) { - UMA_HISTOGRAM_ENUMERATION(kMostUsedContextMenuActionHistogramName, - *most_used_action); - } - const char* total_usage_histogram; const char* total_usage_histogram_to_remove; if (is_premium_) { @@ -430,12 +659,10 @@ void AIChatMetrics::ReportContextMenuMetrics() { kContextMenuPremiumUsageCountHistogramName; } - if (is_enabled_) { - p3a_utils::RecordToHistogramBucket(total_usage_histogram, - kContextMenuUsageBuckets, total_usages); - base::UmaHistogramExactLinear(total_usage_histogram_to_remove, INT_MAX - 1, - 7); - } + p3a_utils::RecordToHistogramBucket(total_usage_histogram, + kContextMenuUsageBuckets, total_usages); + base::UmaHistogramExactLinear(total_usage_histogram_to_remove, INT_MAX - 1, + 7); } void AIChatMetrics::ReportEntryPointUsageMetric() { @@ -443,23 +670,26 @@ void AIChatMetrics::ReportEntryPointUsageMetric() { return; } - uint64_t entry_point_total_max = 0; - std::optional most_used_entry_point; + ReportMostUsedMetric(entry_point_storages_, kMostUsedEntryPointHistogramName, + EntryPoint::kMaxValue); +} - for (int i = 0; i <= static_cast(EntryPoint::kMaxValue); i++) { - EntryPoint entry_point = static_cast(i); - uint64_t entry_point_total = - entry_point_storages_[entry_point]->GetWeeklySum(); - if (entry_point_total > entry_point_total_max) { - most_used_entry_point = entry_point; - entry_point_total_max = entry_point_total; - } +void AIChatMetrics::ReportFullPageUsageMetric() { + if (!is_enabled_) { + return; } - if (most_used_entry_point) { - UMA_HISTOGRAM_ENUMERATION(kMostUsedEntryPointHistogramName, - *most_used_entry_point); + uint64_t sidebar_opens = sidebar_usage_storage_.GetWeeklySum(); + uint64_t full_page_switches = full_page_switch_storage_.GetWeeklySum(); + + if (sidebar_opens == 0) { + return; } + + int percentage = + static_cast(std::ceil((full_page_switches * 100.0) / sidebar_opens)); + p3a_utils::RecordToHistogramBucket(kFullPageSwitchesHistogramName, + kFullPageSwitchesBuckets, percentage); } #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) diff --git a/components/ai_chat/core/browser/ai_chat_metrics.h b/components/ai_chat/core/browser/ai_chat_metrics.h index 8025c64f87b2..1bed3669d2a0 100644 --- a/components/ai_chat/core/browser/ai_chat_metrics.h +++ b/components/ai_chat/core/browser/ai_chat_metrics.h @@ -8,6 +8,7 @@ #include #include +#include #include "base/containers/flat_map.h" #include "base/functional/callback.h" @@ -19,6 +20,7 @@ #include "brave/components/time_period_storage/time_period_storage.h" #include "brave/components/time_period_storage/weekly_storage.h" #include "build/build_config.h" +#include "mojo/public/cpp/bindings/receiver_set.h" class PrefRegistrySimple; class PrefService; @@ -67,6 +69,22 @@ inline constexpr char kChatCountNebulaHistogramName[] = "Brave.AIChat.ChatCount.Nebula"; inline constexpr char kMostUsedEntryPointHistogramName[] = "Brave.AIChat.MostUsedEntryPoint"; +inline constexpr char kFirstChatPromptsHistogramName[] = + "Brave.AIChat.FirstChatPrompts"; +inline constexpr char kChatHistoryUsageHistogramName[] = + "Brave.AIChat.ChatHistoryUsage"; +inline constexpr char kMaxChatDurationHistogramName[] = + "Brave.AIChat.MaxChatDuration"; +inline constexpr char kMostUsedContextSourceHistogramName[] = + "Brave.AIChat.MostUsedContextSource"; +inline constexpr char kUsedConversationStarterHistogramName[] = + "Brave.AIChat.UsedConversationStarter"; +inline constexpr char kFullPageSwitchesHistogramName[] = + "Brave.AIChat.FullPageSwitches"; +inline constexpr char kRateLimitStopsHistogramName[] = + "Brave.AIChat.RateLimitStops"; +inline constexpr char kContextLimitsHistogramName[] = + "Brave.AIChat.ContextLimits"; enum class EntryPoint { kOmniboxItem = 0, @@ -91,27 +109,51 @@ enum class ContextMenuAction { kMaxValue = kChangeLength }; -class AIChatMetrics { +enum class ContextSource { + kOmniboxInput = 0, + kConversationStarter = 1, + kPageSummary = 2, + kTextInputWithPage = 3, + kTextInputWithoutPage = 4, + kTextInputViaFullPage = 5, + kQuickAction = 6, + kMaxValue = kQuickAction +}; + +class ConversationHandlerForMetrics { + public: + virtual ~ConversationHandlerForMetrics() = default; + virtual size_t GetConversationHistorySize() = 0; + virtual bool should_send_page_contents() const = 0; + virtual mojom::APIError current_error() const = 0; +}; + +class AIChatMetrics : public mojom::Metrics { public: using RetrievePremiumStatusCallback = base::OnceCallback; explicit AIChatMetrics(PrefService* local_state); - ~AIChatMetrics(); + ~AIChatMetrics() override; AIChatMetrics(const AIChatMetrics&) = delete; AIChatMetrics& operator=(const AIChatMetrics&) = delete; static void RegisterPrefs(PrefRegistrySimple* registry); + void Bind(mojo::PendingReceiver receiver); + void RecordEnabled( bool is_enabled, bool is_new_user, RetrievePremiumStatusCallback retrieve_premium_status_callback); void RecordReset(); - void RecordNewChat(); - void RecordNewPrompt(); + void RecordNewPrompt(ConversationHandlerForMetrics* handler, + mojom::ConversationPtr& conversation, + mojom::ConversationTurnPtr& entry); + void RecordConversationUnload(std::string_view conversation_uuid); + void RecordConversationsCleared(); #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) void RecordOmniboxOpen(); @@ -119,30 +161,52 @@ class AIChatMetrics { void RecordContextMenuUsage(ContextMenuAction action); void HandleOpenViaEntryPoint(EntryPoint entry_point); + void RecordSidebarUsage(); + void RecordFullPageSwitch(); #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) void OnPremiumStatusUpdated(bool is_enabled, bool is_new_user, mojom::PremiumStatus premium_status, mojom::PremiumInfoPtr); + void MaybeRecordLastError(ConversationHandlerForMetrics* handler); + + // Metrics: + void OnSendingPromptWithFullPage() override; + void OnQuickActionStatusChange(bool is_enabled) override; private: void ReportAllMetrics(); void ReportFeatureUsageMetrics(); void ReportChatCounts(); + void ReportContextSource(); + void ReportFullPageUsage(); + void ReportLimitMetrics(); #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) void ReportOmniboxCounts(); void ReportContextMenuMetrics(); void ReportEntryPointUsageMetric(); + void ReportFullPageUsageMetric(); #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) + void MaybeReportFirstChatPrompts(bool new_prompt_made); + void RecordContextSource(ConversationHandlerForMetrics* handler, + mojom::ConversationTurnPtr& entry); + bool is_enabled_ = false; bool is_premium_ = false; bool premium_check_in_progress_ = false; + bool prompted_via_omnibox_ = false; + bool prompted_via_full_page_ = false; + bool prompted_via_quick_action_ = false; std::optional acquisition_source_ = std::nullopt; WeeklyStorage chat_count_storage_; + WeeklyStorage chat_with_history_count_storage_; + WeeklyStorage chat_durations_storage_; WeeklyStorage prompt_count_storage_; + WeeklyStorage rate_limit_storage_; + WeeklyStorage context_limit_storage_; #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) base::flat_map> context_menu_usage_storages_; @@ -152,14 +216,23 @@ class AIChatMetrics { base::flat_map> entry_point_storages_; + WeeklyStorage sidebar_usage_storage_; + WeeklyStorage full_page_switch_storage_; #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) + base::flat_map> + context_source_storages_; + + base::flat_map conversation_start_times_; base::OneShotTimer report_debounce_timer_; + base::OneShotTimer first_chat_report_debounce_timer_; base::WallClockTimer periodic_report_timer_; raw_ptr local_state_; + mojo::ReceiverSet receivers_; + base::WeakPtrFactory weak_ptr_factory_{this}; }; diff --git a/components/ai_chat/core/browser/ai_chat_metrics_unittest.cc b/components/ai_chat/core/browser/ai_chat_metrics_unittest.cc index 7b1c0c9117ad..ef4c0073afa2 100644 --- a/components/ai_chat/core/browser/ai_chat_metrics_unittest.cc +++ b/components/ai_chat/core/browser/ai_chat_metrics_unittest.cc @@ -18,9 +18,11 @@ #include "base/test/metrics/histogram_tester.h" #include "base/test/task_environment.h" #include "base/time/time.h" +#include "components/grit/brave_components_strings.h" #include "components/prefs/testing_pref_service.h" #include "content/public/test/browser_task_environment.h" #include "testing/gtest/include/gtest/gtest.h" +#include "ui/base/l10n/l10n_util.h" namespace ai_chat { @@ -36,14 +38,15 @@ class AIChatMetricsUnitTest : public testing::Test { ai_chat_metrics_ = std::make_unique(&local_state_); } - void RecordPrompts(bool new_chats, size_t chat_count) { - for (size_t i = 0; i < chat_count; i++) { - if (new_chats) { - ai_chat_metrics_->RecordNewChat(); - } - ai_chat_metrics_->RecordNewPrompt(); + void RecordPrompts(std::string id, size_t prompt_count) { + auto conversation_info = CreateConversationAndTurn(id, "test"); + + for (size_t j = 0; j < prompt_count; j++) { + conversation_handler_.current_history_size_++; + ai_chat_metrics_->RecordNewPrompt(&conversation_handler_, + conversation_info.first, + conversation_info.second); } - task_environment_.FastForwardBy(base::Seconds(5)); } AIChatMetrics::RetrievePremiumStatusCallback GetPremiumCallback() { @@ -57,6 +60,44 @@ class AIChatMetricsUnitTest : public testing::Test { } protected: + class MockConversationHandler : public ConversationHandlerForMetrics { + public: + MockConversationHandler() = default; + ~MockConversationHandler() override = default; + + size_t GetConversationHistorySize() override { + return current_history_size_; + } + bool should_send_page_contents() const override { + return should_send_page_contents_; + } + mojom::APIError current_error() const override { return current_error_; } + + size_t current_history_size_ = 0; + bool should_send_page_contents_ = false; + mojom::APIError current_error_ = mojom::APIError::None; + }; + + std::pair + CreateConversationAndTurn(const std::string& uuid, + const std::string& turn_text) { + auto turn = mojom::ConversationTurn::New(); + turn->uuid = uuid; + turn->character_type = mojom::CharacterType::HUMAN; + turn->action_type = mojom::ActionType::UNSPECIFIED; + turn->text = turn_text; + turn->created_time = base::Time::Now(); + turn->from_brave_search_SERP = false; + + auto conversation = mojom::Conversation::New(); + conversation->uuid = uuid; + conversation->updated_time = turn->created_time; + conversation->has_content = true; + + return {std::move(conversation), std::move(turn)}; + } + + MockConversationHandler conversation_handler_; bool is_premium_ = false; content::BrowserTaskEnvironment task_environment_; TestingPrefServiceSimple local_state_; @@ -93,13 +134,19 @@ TEST_F(AIChatMetricsUnitTest, ChatCount) { ai_chat_metrics_->RecordEnabled(true, false, GetPremiumCallback()); histogram_tester_.ExpectTotalCount(kChatCountHistogramName, 0); - RecordPrompts(true, 1); + RecordPrompts("abc", 5); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectUniqueSample(kChatCountHistogramName, 0, 1); - RecordPrompts(true, 3); + RecordPrompts("def1", 5); + RecordPrompts("def2", 5); + RecordPrompts("def3", 5); + RecordPrompts("def4", 5); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectBucketCount(kChatCountHistogramName, 1, 1); - RecordPrompts(true, 3); + RecordPrompts("xyz1", 3); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectBucketCount(kChatCountHistogramName, 1, 1); histogram_tester_.ExpectBucketCount(kChatCountHistogramName, 2, 1); histogram_tester_.ExpectTotalCount(kChatCountHistogramName, 3); @@ -122,16 +169,20 @@ TEST_F(AIChatMetricsUnitTest, AvgPromptsPerChat) { ai_chat_metrics_->RecordEnabled(true, false, GetPremiumCallback()); histogram_tester_.ExpectTotalCount(kAvgPromptCountHistogramName, 0); - RecordPrompts(true, 1); + RecordPrompts("abc", 1); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectUniqueSample(kAvgPromptCountHistogramName, 0, 1); - RecordPrompts(false, 2); + RecordPrompts("abc", 2); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectBucketCount(kAvgPromptCountHistogramName, 1, 1); - RecordPrompts(false, 4); + RecordPrompts("abc", 4); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectBucketCount(kAvgPromptCountHistogramName, 2, 1); - RecordPrompts(true, 1); + RecordPrompts("def", 1); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectBucketCount(kAvgPromptCountHistogramName, 1, 2); histogram_tester_.ExpectTotalCount(kAvgPromptCountHistogramName, 4); @@ -158,7 +209,8 @@ TEST_F(AIChatMetricsUnitTest, UsageDailyWeeklyAndMonthly) { histogram_tester_.ExpectTotalCount(kUsageWeeklyHistogramName, 0); histogram_tester_.ExpectTotalCount(kUsageMonthlyHistogramName, 0); - RecordPrompts(true, 1); + RecordPrompts("abc", 1); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectUniqueSample(kUsageDailyHistogramName, 1, 1); histogram_tester_.ExpectUniqueSample(kUsageWeeklyHistogramName, 1, 1); histogram_tester_.ExpectUniqueSample(kUsageMonthlyHistogramName, 1, 1); @@ -166,7 +218,8 @@ TEST_F(AIChatMetricsUnitTest, UsageDailyWeeklyAndMonthly) { is_premium_ = true; ai_chat_metrics_->OnPremiumStatusUpdated( true, false, mojom::PremiumStatus::Active, nullptr); - RecordPrompts(true, 1); + RecordPrompts("def", 1); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectBucketCount(kUsageDailyHistogramName, 2, 1); histogram_tester_.ExpectBucketCount(kUsageWeeklyHistogramName, 2, 1); histogram_tester_.ExpectBucketCount(kUsageMonthlyHistogramName, 2, 1); @@ -183,7 +236,8 @@ TEST_F(AIChatMetricsUnitTest, FeatureUsageNotNewUser) { // recorded internally histogram_tester_.ExpectUniqueSample(kNewUserReturningHistogramName, 1, 1); - RecordPrompts(true, 1); + RecordPrompts("abc", 1); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectUniqueSample(kNewUserReturningHistogramName, 1, 2); } @@ -195,7 +249,8 @@ TEST_F(AIChatMetricsUnitTest, FeatureUsage) { histogram_tester_.ExpectUniqueSample(kNewUserReturningHistogramName, 0, 2); histogram_tester_.ExpectTotalCount(kLastUsageTimeHistogramName, 0); - RecordPrompts(true, 1); + RecordPrompts("abc", 1); + task_environment_.FastForwardBy(base::Seconds(5)); histogram_tester_.ExpectBucketCount(kNewUserReturningHistogramName, 2, 1); histogram_tester_.ExpectUniqueSample(kLastUsageTimeHistogramName, 1, 1); @@ -375,4 +430,233 @@ TEST_F(AIChatMetricsUnitTest, Reset) { std::numeric_limits::max() - 1, 1); } +TEST_F(AIChatMetricsUnitTest, ChatHistory) { + ai_chat_metrics_->RecordEnabled(true, false, GetPremiumCallback()); + + // Record some prompts with single turns + RecordPrompts("chat1", 1); + task_environment_.FastForwardBy(base::Seconds(5)); + histogram_tester_.ExpectTotalCount(kChatHistoryUsageHistogramName, 1); + histogram_tester_.ExpectUniqueSample(kChatHistoryUsageHistogramName, 0, 1); + + conversation_handler_.current_history_size_ = 1; + RecordPrompts("chat1", 1); + task_environment_.FastForwardBy(base::Seconds(5)); + + histogram_tester_.ExpectTotalCount(kChatHistoryUsageHistogramName, 2); + histogram_tester_.ExpectUniqueSample(kChatHistoryUsageHistogramName, 0, 2); + + ai_chat_metrics_->RecordConversationUnload("chat1"); + + conversation_handler_.current_history_size_ = 1; + RecordPrompts("chat1", 1); + task_environment_.FastForwardBy(base::Seconds(5)); + + histogram_tester_.ExpectTotalCount(kChatHistoryUsageHistogramName, 3); + histogram_tester_.ExpectBucketCount(kChatHistoryUsageHistogramName, 5, 1); + + task_environment_.FastForwardBy(base::Days(7)); + histogram_tester_.ExpectTotalCount(kChatHistoryUsageHistogramName, 9); + + task_environment_.FastForwardBy(base::Days(7)); + histogram_tester_.ExpectTotalCount(kChatHistoryUsageHistogramName, 9); +} + +TEST_F(AIChatMetricsUnitTest, ChatDuration) { + ai_chat_metrics_->RecordEnabled(true, false, GetPremiumCallback()); + + RecordPrompts("chat1", 1); + task_environment_.FastForwardBy(base::Seconds(5)); + histogram_tester_.ExpectUniqueSample(kMaxChatDurationHistogramName, 0, 1); + + task_environment_.FastForwardBy(base::Minutes(29)); + + RecordPrompts("chat1", 1); + task_environment_.FastForwardBy(base::Seconds(5)); + + histogram_tester_.ExpectBucketCount(kMaxChatDurationHistogramName, 4, 1); + histogram_tester_.ExpectTotalCount(kMaxChatDurationHistogramName, 2); + + task_environment_.FastForwardBy(base::Minutes(15)); + RecordPrompts("chat1", 1); + task_environment_.FastForwardBy(base::Seconds(5)); + + histogram_tester_.ExpectBucketCount(kMaxChatDurationHistogramName, 5, 1); + histogram_tester_.ExpectTotalCount(kMaxChatDurationHistogramName, 3); + + ai_chat_metrics_->RecordConversationUnload("chat1"); + + task_environment_.FastForwardBy(base::Minutes(60)); + RecordPrompts("chat1", 1); + task_environment_.FastForwardBy(base::Seconds(5)); + + histogram_tester_.ExpectBucketCount(kMaxChatDurationHistogramName, 5, 2); + histogram_tester_.ExpectTotalCount(kMaxChatDurationHistogramName, 4); + + task_environment_.FastForwardBy(base::Days(7)); + histogram_tester_.ExpectTotalCount(kMaxChatDurationHistogramName, 10); + + task_environment_.FastForwardBy(base::Days(7)); + histogram_tester_.ExpectTotalCount(kMaxChatDurationHistogramName, 10); +} + +TEST_F(AIChatMetricsUnitTest, FirstChatPrompts) { + ai_chat_metrics_->RecordEnabled(true, false, GetPremiumCallback()); + + histogram_tester_.ExpectTotalCount(kFirstChatPromptsHistogramName, 0); + + RecordPrompts("chat1", 4); + task_environment_.FastForwardBy(base::Seconds(5)); + + histogram_tester_.ExpectTotalCount(kFirstChatPromptsHistogramName, 0); + + ai_chat_metrics_->RecordConversationUnload("chat1"); + task_environment_.FastForwardBy(base::Seconds(5)); + + histogram_tester_.ExpectUniqueSample(kFirstChatPromptsHistogramName, 2, 1); + + ai_chat_metrics_->RecordConversationUnload("chat1"); + RecordPrompts("chat1", 30); + task_environment_.FastForwardBy(base::Seconds(5)); + + histogram_tester_.ExpectUniqueSample(kFirstChatPromptsHistogramName, 2, 1); + + task_environment_.FastForwardBy(base::Days(7)); + histogram_tester_.ExpectTotalCount(kFirstChatPromptsHistogramName, 1); +} + +#if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) +TEST_F(AIChatMetricsUnitTest, FullPageSwitch) { + ai_chat_metrics_->RecordEnabled(true, false, GetPremiumCallback()); + + ai_chat_metrics_->RecordSidebarUsage(); + ai_chat_metrics_->RecordSidebarUsage(); + + histogram_tester_.ExpectUniqueSample(kFullPageSwitchesHistogramName, 0, 2); + + ai_chat_metrics_->RecordFullPageSwitch(); + ai_chat_metrics_->RecordFullPageSwitch(); + + histogram_tester_.ExpectBucketCount(kFullPageSwitchesHistogramName, 3, 1); + histogram_tester_.ExpectBucketCount(kFullPageSwitchesHistogramName, 4, 1); + histogram_tester_.ExpectTotalCount(kFullPageSwitchesHistogramName, 4); + + task_environment_.FastForwardBy(base::Days(7)); + histogram_tester_.ExpectTotalCount(kFullPageSwitchesHistogramName, 10); + + task_environment_.FastForwardBy(base::Days(7)); + histogram_tester_.ExpectTotalCount(kFullPageSwitchesHistogramName, 10); +} +#endif + +TEST_F(AIChatMetricsUnitTest, ContextSource) { + ai_chat_metrics_->RecordEnabled(true, false, GetPremiumCallback()); + + histogram_tester_.ExpectTotalCount(kMostUsedContextSourceHistogramName, 0); + + // Test conversation starter context when history size > 1 + conversation_handler_.current_history_size_ = 2; + auto chat = CreateConversationAndTurn("chat10", "hello"); + chat.second->action_type = mojom::ActionType::CONVERSATION_STARTER; + ai_chat_metrics_->RecordNewPrompt(&conversation_handler_, chat.first, + chat.second); + histogram_tester_.ExpectUniqueSample(kMostUsedContextSourceHistogramName, 4, + 1); + histogram_tester_.ExpectTotalCount(kMostUsedContextSourceHistogramName, 1); + + conversation_handler_.current_history_size_ = 1; + // Test conversation starter context + chat = CreateConversationAndTurn("chat2", "hello"); + chat.second->action_type = mojom::ActionType::CONVERSATION_STARTER; + ai_chat_metrics_->RecordNewPrompt(&conversation_handler_, chat.first, + chat.second); + ai_chat_metrics_->RecordNewPrompt(&conversation_handler_, chat.first, + chat.second); + histogram_tester_.ExpectBucketCount(kMostUsedContextSourceHistogramName, 1, + 2); + + // Test page summary context + chat = CreateConversationAndTurn("chat1", "test"); + chat.second->action_type = mojom::ActionType::SUMMARIZE_PAGE; + for (int i = 0; i < 3; i++) { + ai_chat_metrics_->RecordNewPrompt(&conversation_handler_, chat.first, + chat.second); + } + histogram_tester_.ExpectBucketCount(kMostUsedContextSourceHistogramName, 2, + 1); + + // Test text input with page context + chat = CreateConversationAndTurn("chat4", "test"); + conversation_handler_.should_send_page_contents_ = true; + for (int i = 0; i < 4; i++) { + ai_chat_metrics_->RecordNewPrompt(&conversation_handler_, chat.first, + chat.second); + } + histogram_tester_.ExpectBucketCount(kMostUsedContextSourceHistogramName, 3, + 1); + conversation_handler_.should_send_page_contents_ = false; + + // Test text input without page context (default) + chat = CreateConversationAndTurn("chat6", "test"); + for (int i = 0; i < 4; i++) { + ai_chat_metrics_->RecordNewPrompt(&conversation_handler_, chat.first, + chat.second); + } + histogram_tester_.ExpectBucketCount(kMostUsedContextSourceHistogramName, 4, + 2); + + // Test text input via full page context + chat = CreateConversationAndTurn("chat5", "test"); + for (int i = 0; i < 6; i++) { + ai_chat_metrics_->OnSendingPromptWithFullPage(); + ai_chat_metrics_->RecordNewPrompt(&conversation_handler_, chat.first, + chat.second); + } + histogram_tester_.ExpectBucketCount(kMostUsedContextSourceHistogramName, 5, + 1); + + // Test quick action context + chat = CreateConversationAndTurn("chat3", "test"); + for (size_t i = 0; i < 7; i++) { + ai_chat_metrics_->OnQuickActionStatusChange(true); + ai_chat_metrics_->RecordNewPrompt(&conversation_handler_, chat.first, + chat.second); + } + histogram_tester_.ExpectBucketCount(kMostUsedContextSourceHistogramName, 6, + 1); + +#if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS) + chat = CreateConversationAndTurn("chat2", "test"); + for (size_t i = 0; i < 7; i++) { + ai_chat_metrics_->RecordOmniboxOpen(); + ai_chat_metrics_->RecordNewPrompt(&conversation_handler_, chat.first, + chat.second); + } + histogram_tester_.ExpectBucketCount(kMostUsedContextSourceHistogramName, 0, + 1); +#endif +} + +TEST_F(AIChatMetricsUnitTest, RateLimitMetrics) { + ai_chat_metrics_->RecordEnabled(true, true, GetPremiumCallback()); + + histogram_tester_.ExpectTotalCount(kRateLimitStopsHistogramName, 0); + + RecordPrompts("chat1", 1); + task_environment_.FastForwardBy(base::Seconds(5)); + histogram_tester_.ExpectUniqueSample(kRateLimitStopsHistogramName, 0, 1); + + conversation_handler_.current_error_ = mojom::APIError::RateLimitReached; + ai_chat_metrics_->MaybeRecordLastError(&conversation_handler_); + + histogram_tester_.ExpectBucketCount(kRateLimitStopsHistogramName, 1, 1); + histogram_tester_.ExpectTotalCount(kRateLimitStopsHistogramName, 2); + + task_environment_.FastForwardBy(base::Days(7)); + histogram_tester_.ExpectTotalCount(kRateLimitStopsHistogramName, 8); + + task_environment_.FastForwardBy(base::Days(7)); + histogram_tester_.ExpectTotalCount(kRateLimitStopsHistogramName, 8); +} + } // namespace ai_chat diff --git a/components/ai_chat/core/browser/ai_chat_service.cc b/components/ai_chat/core/browser/ai_chat_service.cc index 30f67c06eaf5..9e4dba658433 100644 --- a/components/ai_chat/core/browser/ai_chat_service.cc +++ b/components/ai_chat/core/browser/ai_chat_service.cc @@ -309,7 +309,7 @@ void AIChatService::DeleteConversations(std::optional begin_time, ReloadConversations(); } if (ai_chat_metrics_ != nullptr) { - ai_chat_metrics_->RecordReset(); + ai_chat_metrics_->RecordConversationsCleared(); } OnConversationListChanged(); return; @@ -584,6 +584,9 @@ void AIChatService::DeleteConversation(const std::string& id) { if (handler_it != conversation_handlers_.end()) { conversation_observations_.RemoveObservation(handler_it->second.get()); conversation_handlers_.erase(id); + if (ai_chat_metrics_) { + ai_chat_metrics_->RecordConversationUnload(id); + } } conversations_.erase(id); DVLOG(1) << "Erased conversation due to deletion request (" << id @@ -653,6 +656,7 @@ void AIChatService::MaybeUnloadConversation( auto uuid = conversation_handler->get_conversation_uuid(); conversation_observations_.RemoveObservation(conversation_handler); conversation_handlers_.erase(uuid); + DVLOG(1) << "Unloaded conversation (" << uuid << ") from memory. Now have " << conversations_.size() << " Conversation metadata items and " << conversation_handlers_.size() @@ -711,6 +715,9 @@ bool AIChatService::IsAIChatHistoryEnabled() { void AIChatService::OnRequestInProgressChanged(ConversationHandler* handler, bool in_progress) { + if (ai_chat_metrics_) { + ai_chat_metrics_->MaybeRecordLastError(handler); + } // We don't unload a conversation if it has a request in progress, so check // again when that changes. if (!in_progress) { @@ -757,8 +764,7 @@ void AIChatService::HandleFirstEntry( // Record metrics if (ai_chat_metrics_ != nullptr) { if (handler->GetConversationHistory().size() == 1) { - ai_chat_metrics_->RecordNewChat(); - ai_chat_metrics_->RecordNewPrompt(); + ai_chat_metrics_->RecordNewPrompt(handler, conversation, entry); } } } @@ -794,7 +800,7 @@ void AIChatService::HandleNewEntry( // Record metrics if (ai_chat_metrics_ != nullptr && entry->character_type == mojom::CharacterType::HUMAN) { - ai_chat_metrics_->RecordNewPrompt(); + ai_chat_metrics_->RecordNewPrompt(handler, conversation, entry); } } @@ -811,6 +817,10 @@ void AIChatService::OnConversationEntryRemoved(ConversationHandler* handler, void AIChatService::OnClientConnectionChanged(ConversationHandler* handler) { DVLOG(4) << "Client connection changed for conversation " << handler->get_conversation_uuid(); + if (ai_chat_metrics_ != nullptr && !handler->IsAnyClientConnected()) { + ai_chat_metrics_->RecordConversationUnload( + handler->get_conversation_uuid()); + } MaybeUnloadConversation(handler); } @@ -877,6 +887,12 @@ void AIChatService::BindConversation( std::move(receiver), std::move(conversation_ui_handler))); } +void AIChatService::BindMetrics(mojo::PendingReceiver metrics) { + if (ai_chat_metrics_) { + ai_chat_metrics_->Bind(std::move(metrics)); + } +} + void AIChatService::BindObserver( mojo::PendingRemote observer, BindObserverCallback callback) { diff --git a/components/ai_chat/core/browser/ai_chat_service.h b/components/ai_chat/core/browser/ai_chat_service.h index 40f9b1416123..9859b012b570 100644 --- a/components/ai_chat/core/browser/ai_chat_service.h +++ b/components/ai_chat/core/browser/ai_chat_service.h @@ -159,6 +159,8 @@ class AIChatService : public KeyedService, mojo::PendingReceiver receiver, mojo::PendingRemote conversation_ui_handler) override; + + void BindMetrics(mojo::PendingReceiver metrics) override; void BindObserver(mojo::PendingRemote ui, BindObserverCallback callback) override; @@ -250,6 +252,8 @@ class AIChatService : public KeyedService, // actively used. Any metadata that needs to stay in-memory // should be kept in |conversations_|. Any other data only for viewing // conversation detail should be persisted to database. + // TODO(djandries): If the above requirement for this map changes, + // adjust the metrics that depend on loaded conversation state accordingly. std::map> conversation_handlers_; diff --git a/components/ai_chat/core/browser/conversation_handler.cc b/components/ai_chat/core/browser/conversation_handler.cc index 208b983c29b6..c18cbe13576a 100644 --- a/components/ai_chat/core/browser/conversation_handler.cc +++ b/components/ai_chat/core/browser/conversation_handler.cc @@ -1799,6 +1799,18 @@ void ConversationHandler::OnStateForConversationEntriesChanged() { } } +size_t ConversationHandler::GetConversationHistorySize() { + return GetConversationHistory().size(); +} + +bool ConversationHandler::should_send_page_contents() const { + return should_send_page_contents_; +} + +mojom::APIError ConversationHandler::current_error() const { + return current_error_; +} + } // namespace ai_chat #undef STARTER_PROMPT diff --git a/components/ai_chat/core/browser/conversation_handler.h b/components/ai_chat/core/browser/conversation_handler.h index 9f91d91c6ae5..11f6375cddb6 100644 --- a/components/ai_chat/core/browser/conversation_handler.h +++ b/components/ai_chat/core/browser/conversation_handler.h @@ -29,6 +29,7 @@ #include "base/task/sequenced_task_runner.h" #include "base/types/expected.h" #include "brave/components/ai_chat/core/browser/ai_chat_credential_manager.h" +#include "brave/components/ai_chat/core/browser/ai_chat_metrics.h" #include "brave/components/ai_chat/core/browser/engine/engine_consumer.h" #include "brave/components/ai_chat/core/browser/model_service.h" #include "brave/components/ai_chat/core/browser/text_embedder.h" @@ -65,7 +66,8 @@ class AIChatCredentialManager; // the in-memory conversation history. class ConversationHandler : public mojom::ConversationHandler, public mojom::UntrustedConversationHandler, - public ModelService::Observer { + public ModelService::Observer, + public ConversationHandlerForMetrics { public: // |invalidation_token| is an optional parameter that will be passed back on // the next call to |GetPageContent| so that the implementer may determine if @@ -288,6 +290,7 @@ class ConversationHandler : public mojom::ConversationHandler, void OnAssociatedContentTitleChanged(); void OnFaviconImageDataChanged(); void OnUserOptedIn(); + size_t GetConversationHistorySize() override; // Some associated content may provide some conversation that the user wants // to continue, e.g. Brave Search. @@ -299,6 +302,10 @@ class ConversationHandler : public mojom::ConversationHandler, std::string get_conversation_uuid() const { return metadata_->uuid; } + bool should_send_page_contents() const override; + + mojom::APIError current_error() const override; + void SetEngineForTesting(std::unique_ptr engine_for_testing) { engine_ = std::move(engine_for_testing); } diff --git a/components/ai_chat/core/common/mojom/ai_chat.mojom b/components/ai_chat/core/common/mojom/ai_chat.mojom index 9ad7a4084447..b201e990b317 100644 --- a/components/ai_chat/core/common/mojom/ai_chat.mojom +++ b/components/ai_chat/core/common/mojom/ai_chat.mojom @@ -352,6 +352,9 @@ interface Service { string conversation_uuid, pending_receiver conversation, pending_remote conversation_ui); + + // Bind metrics interface + BindMetrics(pending_receiver metrics); }; interface ServiceObserver { @@ -554,6 +557,14 @@ interface ConversationUI { OnConversationDeleted(); }; +interface Metrics { + // Notify metrics service that a full page prompt is being sent + OnSendingPromptWithFullPage(); + + // Notify metrics service if a quick action is being used + OnQuickActionStatusChange(bool is_enabled); +}; + [EnableIf=is_android] interface IAPSubscription { GetPurchaseTokenOrderId() => (string token, string order_id); diff --git a/components/ai_chat/core/common/pref_names.h b/components/ai_chat/core/common/pref_names.h index 38a472a8305b..007cfae1c761 100644 --- a/components/ai_chat/core/common/pref_names.h +++ b/components/ai_chat/core/common/pref_names.h @@ -21,6 +21,10 @@ inline constexpr char kBraveChatAutocompleteProviderEnabled[] = "brave.ai_chat.autocomplete_provider_enabled"; inline constexpr char kBraveChatP3AChatCountWeeklyStorage[] = "brave.ai_chat.p3a_chat_count"; +inline constexpr char kBraveChatP3AChatWithHistoryCountWeeklyStorage[] = + "brave.ai_chat.p3a_chat_with_history_count"; +inline constexpr char kBraveChatP3AChatDurationsWeeklyStorage[] = + "brave.ai_chat.p3a_chat_durations"; inline constexpr char kBraveChatP3APromptCountWeeklyStorage[] = "brave.ai_chat.p3a_prompt_count"; // Stores Leo Premium credentials that have already been fetched from the @@ -47,8 +51,22 @@ inline constexpr char kBraveChatP3AUsedSecondDay[] = "brave.ai_chat.p3a_used_second_day"; inline constexpr char kBraveChatP3AContextMenuUsages[] = "brave.ai_chat.p3a_context_menu_usages"; +inline constexpr char kBraveChatP3AEntryPointUsages[] = + "brave.ai_chat.p3a_entry_point_usages"; +inline constexpr char kBraveChatP3ASidebarUsages[] = + "brave.ai_chat.p3a_sidebar_usages"; +inline constexpr char kBraveChatP3AFullPageSwitches[] = + "brave.ai_chat.p3a_full_page_switches"; inline constexpr char kBraveChatP3ALastContextMenuUsageTime[] = "brave.ai_chat.p3a_last_context_menu_time"; +inline constexpr char kBraveChatP3AFirstChatPromptsReported[] = + "brave.ai_chat.p3a_first_chat_prompts_reported"; +inline constexpr char kBraveChatP3AContextSourceUsages[] = + "brave.ai_chat.p3a_context_source_usages"; +inline constexpr char kBraveChatP3ARateLimitStops[] = + "brave.ai_chat.p3a_rate_limit_stops"; +inline constexpr char kBraveChatP3AContextLimits[] = + "brave.ai_chat.p3a_context_limits"; #if BUILDFLAG(IS_ANDROID) inline constexpr char kBraveChatSubscriptionActiveAndroid[] = "brave.ai_chat.subscription_active_android"; diff --git a/components/ai_chat/resources/page/api/index.ts b/components/ai_chat/resources/page/api/index.ts index 3b97e10fe242..5ebbeb7ddd07 100644 --- a/components/ai_chat/resources/page/api/index.ts +++ b/components/ai_chat/resources/page/api/index.ts @@ -43,6 +43,8 @@ class PageAPI extends API { public service: Mojom.ServiceRemote = Mojom.Service.getRemote() + public metrics: Mojom.MetricsRemote = new Mojom.MetricsRemote() + public observer: Mojom.ServiceObserverCallbackRouter = new Mojom.ServiceObserverCallbackRouter() @@ -91,6 +93,8 @@ class PageAPI extends API { allActions }) + this.service.bindMetrics(this.metrics.$.bindNewPipeAndPassReceiver()) + // If we're in standalone mode, listen for tab changes so we can show a picker. if (isStandalone) { Mojom.TabTrackerService.getRemote().addObserver(this.tabObserver.$.bindNewPipeAndPassRemote()) diff --git a/components/ai_chat/resources/page/state/conversation_context.tsx b/components/ai_chat/resources/page/state/conversation_context.tsx index f067a9788984..b5c557d7e472 100644 --- a/components/ai_chat/resources/page/state/conversation_context.tsx +++ b/components/ai_chat/resources/page/state/conversation_context.tsx @@ -415,6 +415,12 @@ export function ConversationContextProvider(props: React.PropsWithChildren) { }) } + React.useEffect(() => { + try { + getAPI().metrics.onQuickActionStatusChange(!!context.selectedActionType) + } catch (e) {} + }, [context.selectedActionType]) + const handleActionTypeClick = (actionType: Mojom.ActionType) => { setPartialContext({ selectedActionType: actionType @@ -461,6 +467,10 @@ export function ConversationContextProvider(props: React.PropsWithChildren) { aiChatContext.dismissStorageNotice() } + if (aiChatContext.isStandalone) { + getAPI().metrics.onSendingPromptWithFullPage() + } + if (context.selectedActionType) { conversationHandler.submitHumanConversationEntryWithAction( context.inputText, diff --git a/components/p3a/metric_names.h b/components/p3a/metric_names.h index faf907c9ce88..0bfcfd667837 100644 --- a/components/p3a/metric_names.h +++ b/components/p3a/metric_names.h @@ -30,17 +30,25 @@ inline constexpr auto kCollectedTypicalHistograms = {"Brave.AIChat.AvgPromptCount", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.ChatCount", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.ChatCount.Nebula", MetricConfig{.ephemeral = true,.nebula = true}}, + {"Brave.AIChat.ChatHistoryUsage", MetricConfig{.ephemeral = true}}, + {"Brave.AIChat.ContextLimits", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.ContextMenu.FreeUsages", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.ContextMenu.MostUsedAction", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.ContextMenu.PremiumUsages", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.Enabled.2", {}}, {"Brave.AIChat.Enabled.SidebarEnabledA", {}}, + {"Brave.AIChat.FirstChatPrompts", MetricConfig{.ephemeral = true}}, + {"Brave.AIChat.FullPageSwitches", MetricConfig{.ephemeral = true}}, + {"Brave.AIChat.MaxChatDuration", MetricConfig{.ephemeral = true}}, + {"Brave.AIChat.MostUsedContextSource", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.MostUsedEntryPoint", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.NewUserReturning", {}}, {"Brave.AIChat.OmniboxOpens", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.OmniboxWeekCompare", MetricConfig{.ephemeral = true}}, + {"Brave.AIChat.RateLimitStops", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.UsageWeekly", MetricConfig{.ephemeral = true}}, {"Brave.AIChat.UsageWeekly.SidebarEnabledA", MetricConfig{.ephemeral = true}}, + {"Brave.AIChat.UsedConversationStarter", {}}, {"Brave.Ads.ClearData", MetricConfig{.ephemeral = true}}, {"Brave.Core.BookmarkCount", {}}, {"Brave.Core.CrashReportsEnabled", {}},