Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: make it running without GCP env #1471

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions lana/app/src/report/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ pub enum ReportError {
JobError(#[from] crate::job::error::JobError),
#[error("ReportError - StorageError: {0}")]
StorageError(#[from] crate::storage::StorageError),
#[error("ReportError - ServiceAccountError: {0}")]
ServiceAccountError(#[from] crate::service_account::error::ServiceAccountError),
}

es_entity::from_es_entity_error!(ReportError);
12 changes: 6 additions & 6 deletions lana/app/src/report/upload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,9 @@ pub(super) mod bq {
pub(super) async fn find_report_outputs(
config: &ReportConfig,
) -> Result<Vec<String>, ReportError> {
let client =
Client::from_service_account_key(config.service_account().service_account_key(), false)
.await?;
let sa_key = config.service_account().service_account_key()?;

let client = Client::from_service_account_key(sa_key, false).await?;
let tables = client
.table()
.list(
Expand All @@ -129,9 +129,9 @@ pub(super) mod bq {
config: &ReportConfig,
report: &str,
) -> Result<Vec<QueryRow>, ReportError> {
let client =
Client::from_service_account_key(config.service_account().service_account_key(), false)
.await?;
let sa_key = config.service_account().service_account_key()?;

let client = Client::from_service_account_key(sa_key, false).await?;
let gcp_project = &config.service_account().gcp_project;
let query = format!(
"SELECT * FROM `{}.{}.{}`",
Expand Down
2 changes: 2 additions & 0 deletions lana/app/src/service_account/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,6 @@ pub enum ServiceAccountError {
Deserialization(#[from] serde_json::Error),
#[error("ServiceAccountError - Base64Decode: {0}")]
Base64Decode(#[from] base64::DecodeError),
#[error("ServiceAccountError - CredentialsNotProvided")]
CredentialsNotProvided,
}
38 changes: 26 additions & 12 deletions lana/app/src/service_account/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
mod error;
pub mod error;

use error::ServiceAccountError;
use gcp_bigquery_client::yup_oauth2::ServiceAccountKey;
Expand All @@ -9,7 +9,7 @@ pub struct ServiceAccountConfig {
#[serde(skip)]
pub gcp_project: String,
#[serde(skip)]
pub sa_creds_base64: String,
pub sa_creds_base64: Option<String>,
#[serde(skip)]
service_account_key: Option<ServiceAccountKey>,

Expand All @@ -21,7 +21,7 @@ impl Default for ServiceAccountConfig {
fn default() -> Self {
Self {
gcp_project: "".to_string(),
sa_creds_base64: "".to_string(),
sa_creds_base64: None,
service_account_key: None,
gcp_location: default_gcp_location(),
}
Expand All @@ -31,8 +31,14 @@ impl Default for ServiceAccountConfig {
impl ServiceAccountConfig {
pub fn set_sa_creds_base64(
mut self,
sa_creds_base64: String,
sa_creds_base64: Option<String>,
) -> Result<Self, ServiceAccountError> {
if sa_creds_base64.is_none() {
return Ok(self);
}

println!("sa_creds_base64: {:?}", sa_creds_base64);

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this intended ?

self.sa_creds_base64 = sa_creds_base64;

let creds = self.get_json_creds()?;
Expand All @@ -48,19 +54,27 @@ impl ServiceAccountConfig {
Ok(self)
}

pub fn service_account_key(&self) -> ServiceAccountKey {
pub fn service_account_key(&self) -> Result<ServiceAccountKey, ServiceAccountError> {
self.service_account_key
.clone()
.expect("Service Account not set")
.as_ref()
.cloned()
.ok_or(ServiceAccountError::CredentialsNotProvided)
}

pub fn get_json_creds(&self) -> Result<String, ServiceAccountError> {
fn get_json_creds(&self) -> Result<String, ServiceAccountError> {
if self.sa_creds_base64.is_none() {
return Err(ServiceAccountError::CredentialsNotProvided);
}

let creds = self
Comment on lines +65 to +68
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

redundant since ok_or on L 72 already handles this

.sa_creds_base64
.as_ref()
.ok_or(ServiceAccountError::CredentialsNotProvided)?
.as_bytes();

use base64::{engine::general_purpose, Engine as _};

Ok(std::str::from_utf8(
&general_purpose::STANDARD.decode(self.sa_creds_base64.as_bytes())?,
)?
.to_string())
Ok(std::str::from_utf8(&general_purpose::STANDARD.decode(creds)?)?.to_string())
}
}

Expand Down
12 changes: 7 additions & 5 deletions lana/app/tests/gcs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@ use lana_app::{

#[tokio::test]
async fn upload_doc() -> anyhow::Result<()> {
let sa_creds_base64 = if let Ok(sa_creds_base64) = std::env::var("SA_CREDS_BASE64") {
sa_creds_base64
} else {
return Ok(());
let sa_creds_base64 = match std::env::var("SA_CREDS_BASE64") {
Ok(value) if !value.trim().is_empty() => value,
_ => {
println!("Skipping GCS test: SA_CREDS_BASE64 not set or empty");
return Ok(());
}
};

let sa = ServiceAccountConfig::default().set_sa_creds_base64(sa_creds_base64)?;
let sa = ServiceAccountConfig::default().set_sa_creds_base64(Some(sa_creds_base64))?;

let config = if let Ok(name_prefix) = std::env::var("DEV_ENV_NAME_PREFIX") {
StorageConfig::new_dev_mode(name_prefix, sa)
Expand Down
2 changes: 1 addition & 1 deletion lana/cli/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ pub struct EnvSecrets {
pub pg_con: String,
pub sumsub_key: String,
pub sumsub_secret: String,
pub sa_creds_base64: String,
pub sa_creds_base64: Option<String>,
}

impl Config {
Expand Down
10 changes: 8 additions & 2 deletions lana/cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,21 +35,27 @@ struct Cli {
#[clap(env = "SUMSUB_SECRET", default_value = "")]
sumsub_secret: String,
#[clap(env = "SA_CREDS_BASE64", default_value = "")]
sa_creds_base64: String,
sa_creds_base64_raw: String,
#[clap(env = "DEV_ENV_NAME_PREFIX")]
dev_env_name_prefix: Option<String>,
}

pub async fn run() -> anyhow::Result<()> {
let cli = Cli::parse();

let sa_creds_base64 = if cli.sa_creds_base64_raw.is_empty() {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That this is optional should be handled in the cli field / clap attribute.

None
} else {
Some(cli.sa_creds_base64_raw)
};

let config = Config::init(
cli.config,
EnvSecrets {
pg_con: cli.pg_con,
sumsub_key: cli.sumsub_key,
sumsub_secret: cli.sumsub_secret,
sa_creds_base64: cli.sa_creds_base64,
sa_creds_base64,
},
cli.dev_env_name_prefix,
)?;
Expand Down
Loading