Skip to content

Commit

Permalink
chore(llmobs): fix telemetry namespace for ragas (#11279)
Browse files Browse the repository at this point in the history
`ml_obs` is the correct namespace for llmobs that's been implemented in
the telemetry backend

update ragas telemetry metrics to use that namespace

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Co-authored-by: lievan <[email protected]>
  • Loading branch information
lievan and lievan authored Nov 6, 2024
1 parent 19f4c58 commit 99431da
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 5 deletions.
5 changes: 3 additions & 2 deletions ddtrace/llmobs/_evaluators/ragas/faithfulness.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

from ddtrace.internal.logger import get_logger
from ddtrace.internal.telemetry import telemetry_writer
from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT
from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL
from ddtrace.internal.utils.version import parse_version
from ddtrace.llmobs._constants import RAGAS_ML_APP_PREFIX
Expand Down Expand Up @@ -138,7 +139,7 @@ def __init__(self, llmobs_service):
raise NotImplementedError("Failed to load dependencies for `ragas_faithfulness` evaluator") from e
finally:
telemetry_writer.add_count_metric(
namespace="llmobs",
namespace=TELEMETRY_APM_PRODUCT.LLMOBS,
name="evaluators.init",
value=1,
tags=(
Expand All @@ -164,7 +165,7 @@ def run_and_submit_evaluation(self, span_event: dict):
return
score_result_or_failure = self.evaluate(span_event)
telemetry_writer.add_count_metric(
"llmobs",
TELEMETRY_APM_PRODUCT.LLMOBS, # type: ignore
"evaluators.run",
1,
tags=(
Expand Down
3 changes: 2 additions & 1 deletion ddtrace/llmobs/_evaluators/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from ddtrace.internal.logger import get_logger
from ddtrace.internal.periodic import PeriodicService
from ddtrace.internal.telemetry import telemetry_writer
from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT
from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator
from ddtrace.llmobs._evaluators.sampler import EvaluatorRunnerSampler

Expand Down Expand Up @@ -55,7 +56,7 @@ def __init__(self, interval: float, llmobs_service=None, evaluators=None):
raise e
finally:
telemetry_writer.add_count_metric(
namespace="llmobs",
namespace=TELEMETRY_APM_PRODUCT.LLMOBS, # type: ignore
name="evaluators.init",
value=1,
tags=(
Expand Down
5 changes: 3 additions & 2 deletions ddtrace/llmobs/_evaluators/sampler.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from ddtrace import config
from ddtrace.internal.logger import get_logger
from ddtrace.internal.telemetry import telemetry_writer
from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT
from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL
from ddtrace.sampling_rule import SamplingRule

Expand Down Expand Up @@ -66,7 +67,7 @@ def parsing_failed_because(msg, maybe_throw_this):
TELEMETRY_LOG_LEVEL.ERROR, message="Evaluator sampling parsing failure because: {}".format(msg)
)
telemetry_writer.add_count_metric(
namespace="llmobs",
namespace=TELEMETRY_APM_PRODUCT.LLMOBS,
name="evaluators.error",
value=1,
tags=(("reason", "sampling_rule_parsing_failure"),),
Expand Down Expand Up @@ -103,7 +104,7 @@ def parsing_failed_because(msg, maybe_throw_this):
span_name = rule.get(EvaluatorRunnerSamplingRule.SPAN_NAME_KEY, SamplingRule.NO_RULE)
evaluator_label = rule.get(EvaluatorRunnerSamplingRule.EVALUATOR_LABEL_KEY, SamplingRule.NO_RULE)
telemetry_writer.add_distribution_metric(
"llmobs",
TELEMETRY_APM_PRODUCT.LLMOBS, # type: ignore
"evaluators.rule_sample_rate",
sample_rate,
tags=(("evaluator_label", evaluator_label), ("span_name", span_name)),
Expand Down

0 comments on commit 99431da

Please sign in to comment.