Skip to content

fix(issues): Replace 'Internal Error' message on Postgres query timeout errors #87691

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 24, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions src/sentry/api/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@
from datetime import timedelta
from typing import Any, Literal, overload

import psycopg2.errorcodes
import sentry_sdk
from django.conf import settings
from django.db.utils import OperationalError
from django.http import HttpRequest
from django.utils import timezone
from rest_framework.exceptions import APIException, ParseError, Throttled
Expand Down Expand Up @@ -423,6 +425,15 @@ def handle_query_errors() -> Generator[None]:
else:
sentry_sdk.capture_exception(error)
raise APIException(detail=message)
except OperationalError as error:
if hasattr(error, "pgcode") and error.pgcode == psycopg2.errorcodes.QUERY_CANCELED:
if options.get("api.postgres-query-timeout-error-handling.enabled"):
sentry_sdk.set_tag("query.error_reason", "Postgres statement timeout")
raise Throttled(
detail="Query timeout. Please try with a smaller date range or fewer conditions."
)
# Let other OperationalErrors propagate as normal
raise


def update_snuba_params_with_timestamp(
Expand Down
8 changes: 8 additions & 0 deletions src/sentry/options/defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -2773,6 +2773,14 @@
flags=FLAG_ALLOW_EMPTY | FLAG_AUTOMATOR_MODIFIABLE,
)

# Killswitch for Postgres query timeout error handling
register(
"api.postgres-query-timeout-error-handling.enabled",
default=False,
type=Bool,
flags=FLAG_MODIFIABLE_BOOL | FLAG_AUTOMATOR_MODIFIABLE,
)

# TODO: remove once removed from options
register(
"issue_platform.use_kafka_partition_key",
Expand Down
44 changes: 41 additions & 3 deletions tests/sentry/api/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@
import unittest
from unittest.mock import MagicMock, patch

import psycopg2
import pytest
from django.db import OperationalError
from django.utils import timezone
from rest_framework.exceptions import APIException
from rest_framework.exceptions import APIException, Throttled
from sentry_sdk import Scope

from sentry.api.utils import (
Expand All @@ -17,6 +19,7 @@
from sentry.exceptions import IncompatibleMetricsQuery, InvalidParams, InvalidSearchQuery
from sentry.testutils.cases import APITestCase
from sentry.testutils.helpers.datetime import freeze_time
from sentry.testutils.helpers.options import override_options
from sentry.utils.snuba import (
DatasetSelectionError,
QueryConnectionFailed,
Expand Down Expand Up @@ -168,13 +171,12 @@ class FooBarError(Exception):
pass


class HandleQueryErrorsTest:
class HandleQueryErrorsTest(APITestCase):
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like this test wasn't running at all from what I can tell since it didnt inherit APITestCase

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice catch!

@patch("sentry.api.utils.ParseError")
def test_handle_query_errors(self, mock_parse_error):
exceptions = [
DatasetSelectionError,
IncompatibleMetricsQuery,
InvalidParams,
Copy link
Member Author

@yuvmen yuvmen Mar 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

InvalidParams is not being converted in the actual code, and fails since now I actually run this test. From what I could tell InvalidParams is actually correctly not being converted to a user error since it is sometimes indeed an 'Internal Error' so I removed it from this test (which didnt run until now).

InvalidSearchQuery,
QueryConnectionFailed,
QueryExecutionError,
Expand All @@ -198,6 +200,42 @@ def test_handle_query_errors(self, mock_parse_error):
except Exception as e:
assert isinstance(e, (FooBarError, APIException))

def test_handle_postgres_timeout(self):
class TimeoutError(OperationalError):
pgcode = psycopg2.errorcodes.QUERY_CANCELED

# Test when option is disabled (default)
try:
with handle_query_errors():
raise TimeoutError()
except Exception as e:
assert isinstance(e, TimeoutError) # Should propagate original error

# Test when option is enabled
with override_options({"api.postgres-query-timeout-error-handling.enabled": True}):
try:
with handle_query_errors():
raise TimeoutError()
except Exception as e:
assert isinstance(e, Throttled)
assert (
str(e)
== "Query timeout. Please try with a smaller date range or fewer conditions."
)

@patch("sentry.api.utils.ParseError")
def test_handle_other_operational_error(self, mock_parse_error):
class OtherError(OperationalError):
# No pgcode attribute
pass

try:
with handle_query_errors():
raise OtherError()
except Exception as e:
assert isinstance(e, OtherError) # Should propagate original error
mock_parse_error.assert_not_called()


class ClampDateRangeTest(unittest.TestCase):
def test_no_clamp_if_range_under_max(self):
Expand Down
Loading