Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(healthcheck): do not return true as response if the check if not applicable #3551

Merged
merged 5 commits into from
Feb 13, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion crates/api_models/src/health_check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
pub struct RouterHealthCheckResponse {
pub database: bool,
pub redis: bool,
pub locker: bool,
pub vault: bool,
#[cfg(feature = "olap")]
pub analytics: bool,
pub outgoing_request: bool,
Expand All @@ -17,4 +17,19 @@ pub struct SchedulerHealthCheckResponse {
pub outgoing_request: bool,
}

pub enum HealthState {
Running,
Error,
NotApplicable,
}

impl From<HealthState> for bool {
fn from(value: HealthState) -> Self {
match value {
HealthState::Running => true,
HealthState::Error | HealthState::NotApplicable => false,
}
}
}

impl common_utils::events::ApiEventMetric for SchedulerHealthCheckResponse {}
52 changes: 33 additions & 19 deletions crates/router/src/core/health_check.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#[cfg(feature = "olap")]
use analytics::health_check::HealthCheck;
use api_models::health_check::HealthState;
use error_stack::ResultExt;
use router_env::logger;

Expand All @@ -12,23 +13,27 @@ use crate::{

#[async_trait::async_trait]
pub trait HealthCheckInterface {
async fn health_check_db(&self) -> CustomResult<(), errors::HealthCheckDBError>;
async fn health_check_redis(&self) -> CustomResult<(), errors::HealthCheckRedisError>;
async fn health_check_locker(&self) -> CustomResult<(), errors::HealthCheckLockerError>;
async fn health_check_outgoing(&self) -> CustomResult<(), errors::HealthCheckOutGoing>;
async fn health_check_db(&self) -> CustomResult<HealthState, errors::HealthCheckDBError>;
async fn health_check_redis(&self) -> CustomResult<HealthState, errors::HealthCheckRedisError>;
async fn health_check_locker(
&self,
) -> CustomResult<HealthState, errors::HealthCheckLockerError>;
async fn health_check_outgoing(&self)
-> CustomResult<HealthState, errors::HealthCheckOutGoing>;
#[cfg(feature = "olap")]
async fn health_check_analytics(&self) -> CustomResult<(), errors::HealthCheckDBError>;
async fn health_check_analytics(&self)
-> CustomResult<HealthState, errors::HealthCheckDBError>;
}

#[async_trait::async_trait]
impl HealthCheckInterface for app::AppState {
async fn health_check_db(&self) -> CustomResult<(), errors::HealthCheckDBError> {
async fn health_check_db(&self) -> CustomResult<HealthState, errors::HealthCheckDBError> {
let db = &*self.store;
db.health_check_db().await?;
Ok(())
Ok(HealthState::Running)
}

async fn health_check_redis(&self) -> CustomResult<(), errors::HealthCheckRedisError> {
async fn health_check_redis(&self) -> CustomResult<HealthState, errors::HealthCheckRedisError> {
let db = &*self.store;
let redis_conn = db
.get_redis_conn()
Expand All @@ -55,10 +60,12 @@ impl HealthCheckInterface for app::AppState {

logger::debug!("Redis delete_key was successful");

Ok(())
Ok(HealthState::Running)
}

async fn health_check_locker(&self) -> CustomResult<(), errors::HealthCheckLockerError> {
async fn health_check_locker(
&self,
) -> CustomResult<HealthState, errors::HealthCheckLockerError> {
let locker = &self.conf.locker;
if !locker.mock_locker {
let mut url = locker.host_rs.to_owned();
Expand All @@ -67,16 +74,19 @@ impl HealthCheckInterface for app::AppState {
services::call_connector_api(self, request)
.await
.change_context(errors::HealthCheckLockerError::FailedToCallLocker)?
.ok();
.map_err(|_| {
error_stack::report!(errors::HealthCheckLockerError::FailedToCallLocker)
})?;
Ok(HealthState::Running)
} else {
Ok(HealthState::NotApplicable)
}

logger::debug!("Locker call was successful");

Ok(())
}

#[cfg(feature = "olap")]
async fn health_check_analytics(&self) -> CustomResult<(), errors::HealthCheckDBError> {
async fn health_check_analytics(
&self,
) -> CustomResult<HealthState, errors::HealthCheckDBError> {
let analytics = &self.pool;
match analytics {
analytics::AnalyticsProvider::Sqlx(client) => client
Expand Down Expand Up @@ -107,10 +117,14 @@ impl HealthCheckInterface for app::AppState {
.await
.change_context(errors::HealthCheckDBError::ClickhouseAnalyticsError)
}
}
}?;

Ok(HealthState::Running)
}

async fn health_check_outgoing(&self) -> CustomResult<(), errors::HealthCheckOutGoing> {
async fn health_check_outgoing(
&self,
) -> CustomResult<HealthState, errors::HealthCheckOutGoing> {
let request = services::Request::new(services::Method::Get, consts::OUTGOING_CALL_URL);
services::call_connector_api(self, request)
.await
Expand All @@ -125,6 +139,6 @@ impl HealthCheckInterface for app::AppState {
})?;

logger::debug!("Outgoing request successful");
Ok(())
Ok(HealthState::Running)
}
}
78 changes: 31 additions & 47 deletions crates/router/src/routes/health.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ async fn deep_health_check_func(state: app::AppState) -> RouterResponse<RouterHe

logger::debug!("Database health check begin");

let db_status = state.health_check_db().await.map(|_| true).map_err(|err| {
let db_status = state.health_check_db().await.map_err(|err| {
error_stack::report!(errors::ApiErrorResponse::HealthCheckError {
component: "Database",
message: err.to_string()
Expand All @@ -56,64 +56,48 @@ async fn deep_health_check_func(state: app::AppState) -> RouterResponse<RouterHe

logger::debug!("Redis health check begin");

let redis_status = state
.health_check_redis()
.await
.map(|_| true)
.map_err(|err| {
error_stack::report!(errors::ApiErrorResponse::HealthCheckError {
component: "Redis",
message: err.to_string()
})
})?;
let redis_status = state.health_check_redis().await.map_err(|err| {
error_stack::report!(errors::ApiErrorResponse::HealthCheckError {
component: "Redis",
message: err.to_string()
})
})?;

logger::debug!("Redis health check end");

logger::debug!("Locker health check begin");

let locker_status = state
.health_check_locker()
.await
.map(|_| true)
.map_err(|err| {
error_stack::report!(errors::ApiErrorResponse::HealthCheckError {
component: "Locker",
message: err.to_string()
})
})?;
let locker_status = state.health_check_locker().await.map_err(|err| {
error_stack::report!(errors::ApiErrorResponse::HealthCheckError {
component: "Locker",
message: err.to_string()
})
})?;

#[cfg(feature = "olap")]
let analytics_status = state
.health_check_analytics()
.await
.map(|_| true)
.map_err(|err| {
error_stack::report!(errors::ApiErrorResponse::HealthCheckError {
component: "Analytics",
message: err.to_string()
})
})?;

let outgoing_check = state
.health_check_outgoing()
.await
.map(|_| true)
.map_err(|err| {
error_stack::report!(errors::ApiErrorResponse::HealthCheckError {
component: "Outgoing Request",
message: err.to_string()
})
})?;
let analytics_status = state.health_check_analytics().await.map_err(|err| {
error_stack::report!(errors::ApiErrorResponse::HealthCheckError {
component: "Analytics",
message: err.to_string()
})
})?;

let outgoing_check = state.health_check_outgoing().await.map_err(|err| {
error_stack::report!(errors::ApiErrorResponse::HealthCheckError {
component: "Outgoing Request",
message: err.to_string()
})
})?;

logger::debug!("Locker health check end");

let response = RouterHealthCheckResponse {
database: db_status,
redis: redis_status,
locker: locker_status,
database: db_status.into(),
redis: redis_status.into(),
vault: locker_status.into(),
#[cfg(feature = "olap")]
analytics: analytics_status,
outgoing_request: outgoing_check,
analytics: analytics_status.into(),
outgoing_request: outgoing_check.into(),
};

Ok(api::ApplicationResponse::Json(response))
Expand Down
Loading