simple config, env and cli (#247)

* simple config, env and cli

* fix comp

* init env first

* added env override

* postgres config

* fmt

* updated env in readme

* examples

* a

.env.example

@@ -0,0 +1,34 @@
+## LiteRpc 
+LITE_RPC_HTTP_ADDR=http://0.0.0.0:8890
+LITE_RPC_WS_ADDR=[::]:8891
+
+## RPC
+RPC_ADDR=http://0.0.0.0:8899
+WS_ADDR=ws://0.0.0.0:8900
+# IDENTITY=your_identity_keypair_here
+
+## Prometheus
+# PROMETHEUS_ADDR=your_prometheus_address_here
+
+## Fanout size and retries configuration
+FANOUT_SIZE=18
+MAX_RETRIES=40
+RETRY_TIMEOUT=3
+
+## Quic Proxy
+# QUIC_PROXY_ADDR=your_quic_proxy_address_here
+
+## gRPC Configuration
+USE_GRPC=false
+GRPC_ADDR=http://127.0.0.0:10000
+# GRPC_X_TOKEN=your_grpc_token_here
+
+## Postgres Configuration
+# PG_ENABLED=true
+# PG_CONFIG=your_postgres_config_here
+# CA_PEM_B64=your_base64_encoded_ca_pem_here
+# CLIENT_PKS_B64=your_base64_encoded_client_pks_here
+# CLIENT_PKS_PASS=your_client_pks_password_here
+
+## Gso
+# DISABLE_GSO=your_disable_gso_setting_here

README.md

@@ -58,17 +58,24 @@ Find a new file named `metrics.csv` in the project root.
 
 ### Environment Variables
 
-| env               | purpose                                        | required?           |
-|-------------------|------------------------------------------------|---------------------|
-| `RPC_URL`         | HTTP URL for a full RPC node                   | yes, for docker     |
-| `WS_URL`          | WS URL for a full RPC node                     | yes, for docker     |
-| `IDENTITY`        | Staked validator identity keypair              | no                  |
-| `PG_ENABLED`      | Set to anything but 'false' to enable Postgres | no                  |
-| `PG_CONFIG`       | Postgres Connection Config                     | if postgres enabled |
-| `CA_PEM_B64`      | Base64 encoded `ca.pem`                        | if postgres enabled |
-| `CLIENT_PKS_B64`  | Base64 encoded `client.pks`                    | if postgres enabled |
-| `CLIENT_PKS_PASS` | Password to `client.pks`                       | if postgres enabled |
-| `DISABLE_GSO`     | Disable GSO completely                         | no                  |
+Thank you for providing the default values. Here's the updated table with the default values for the environment variables based on the additional information:
+
+| Environment Variable     | Purpose                                                          | Required?           | Default Value                                  |
+|--------------------------|------------------------------------------------------------------|---------------------|------------------------------------------------|
+| `RPC_ADDR`               | Address for the RPC node                                         | Replaces default if set | `http://0.0.0.0:8899` (from `DEFAULT_RPC_ADDR`) |
+| `WS_ADDR`                | WebSocket address for the RPC node                               | Replaces default if set | `ws://0.0.0.0:8900` (from `DEFAULT_WS_ADDR`)   |
+| `LITE_RPC_HTTP_ADDR`     | HTTP address for the lite RPC node                               | Replaces default if set | `http://0.0.0.0:8890` (from `DEFAULT_LITE_RPC_ADDR`) |
+| `LITE_RPC_WS_ADDR`       | WebSocket address for the lite RPC node                          | Replaces default if set | `[::]:8891` (from `Config::default_lite_rpc_ws_addr`) |
+| `FANOUT_SIZE`            | Configuration for the fanout size                                | Replaces default if set | `18` (from `DEFAULT_FANOUT_SIZE`)             |
+| `IDENTITY`               | Identity keypair                                                 | Optional, replaces default if set | None |
+| `PROMETHEUS_ADDR`        | Address for Prometheus monitoring                                | Replaces default if set | None specified in provided defaults |
+| `MAX_RETRIES`            | Maximum number of retries per transaction                        | Replaces default if set | `40` (from `MAX_RETRIES`)                     |
+| `RETRY_TIMEOUT`          | Timeout for transaction retries in seconds                       | Replaces default if set | `3` (from `DEFAULT_RETRY_TIMEOUT`)            |
+| `QUIC_PROXY_ADDR`        | Address for QUIC proxy                                           | Optional | None |
+| `USE_GRPC`               | Flag to enable or disable gRPC                                   | Enables gRPC if set | `false` |
+| `GRPC_ADDR`              | gRPC address                                                     | Replaces default if set | `http://127.0.0.0:10000` (from `DEFAULT_GRPC_ADDR`) |
+| `GRPC_X_TOKEN`           | Token for gRPC authentication                                    | Optional | None |
+| `PG_*`                   | Various environment variables for Postgres configuration         | Depends on Postgres usage | Based on `PostgresSessionConfig::new_from_env()` |
 
 ### Postgres
 lite-rpc implements an optional postgres service that can write to postgres

config.example.json

@@ -0,0 +1,23 @@
+{
+  "rpc_addr": "http://0.0.0.0:8899",
+  "ws_addr": "ws://0.0.0.0:8900",
+  "lite_rpc_http_addr": "http://0.0.0.0:8890",
+  "lite_rpc_ws_addr": "[::]:8891",
+  "fanout_size": 18,
+  "identity_keypair": null,
+  "prometheus_addr": "[::]:9091",
+  "maximum_retries_per_tx": 40,
+  "transaction_retry_after_secs": 3,
+  "quic_proxy_addr": null,
+  "use_grpc": false,
+  "grpc_addr": "http://127.0.0.0:10000",
+  "grpc_x_token": null,
+  "postgres": {
+    "pg_config": "your_postgres_config",
+    "ssl": {
+      "ca_pem_b64": "your_base64_encoded_ca_pem",
+      "client_pks_b64": "your_base64_encoded_client_pks",
+      "client_pks_pass": "your_client_pks_password"
+    }
+  }
+}

core/src/keypair_loader.rs

@@ -1,26 +1,25 @@
+use anyhow::Context;
 use solana_sdk::signature::Keypair;
 use std::env;
 
 // note this is duplicated from lite-rpc module
-pub async fn load_identity_keypair(identity_from_cli: &String) -> Option<Keypair> {
-    if let Ok(identity_env_var) = env::var("IDENTITY") {
-        if let Ok(identity_bytes) = serde_json::from_str::<Vec<u8>>(identity_env_var.as_str()) {
-            Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap())
-        } else {
-            // must be a file
-            let identity_file = tokio::fs::read_to_string(identity_env_var.as_str())
-                .await
-                .expect("Cannot find the identity file provided");
-            let identity_bytes: Vec<u8> = serde_json::from_str(&identity_file).unwrap();
-            Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap())
-        }
-    } else if identity_from_cli.is_empty() {
-        None
-    } else {
-        let identity_file = tokio::fs::read_to_string(identity_from_cli.as_str())
+pub async fn load_identity_keypair(
+    identity_from_cli: Option<String>,
+) -> anyhow::Result<Option<Keypair>> {
+    let identity_str = if let Some(identity_from_cli) = identity_from_cli {
+        tokio::fs::read_to_string(identity_from_cli)
             .await
-            .expect("Cannot find the identity file provided");
-        let identity_bytes: Vec<u8> = serde_json::from_str(&identity_file).unwrap();
-        Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap())
-    }
+            .context("Cannot find the identity file provided")?
+    } else if let Ok(identity_env_var) = env::var("IDENTITY") {
+        identity_env_var
+    } else {
+        return Ok(None);
+    };
+
+    let identity_bytes: Vec<u8> =
+        serde_json::from_str(&identity_str).context("Invalid identity format expected Vec<u8>")?;
+
+    Ok(Some(
+        Keypair::from_bytes(identity_bytes.as_slice()).context("Invalid identity")?,
+    ))
 }

history/Cargo.toml

@@ -26,4 +26,5 @@ chrono = {workspace = true}
 bincode = {workspace = true}
 base64 = {workspace = true}
 itertools = {workspace = true}
-tokio-postgres = { version = "0.7.8", features = ["with-chrono-0_4"] }
+tokio-postgres = { version = "0.7.8", features = ["with-chrono-0_4"] }
+serde = { workspace = true }

history/src/postgres/mod.rs

@@ -1,3 +1,4 @@
 pub mod postgres_block;
+pub mod postgres_config;
 pub mod postgres_session;
 pub mod postgres_transaction;

history/src/postgres/postgres_config.rs

@@ -0,0 +1,52 @@
+use anyhow::Context;
+use std::env;
+use tokio_postgres::config::SslMode;
+
+#[derive(serde::Deserialize, Debug, Clone)]
+pub struct PostgresSessionConfig {
+    pub pg_config: String,
+    pub ssl: Option<PostgresSessionSslConfig>,
+}
+
+#[derive(serde::Deserialize, Debug, Clone)]
+pub struct PostgresSessionSslConfig {
+    pub ca_pem_b64: String,
+    pub client_pks_b64: String,
+    pub client_pks_pass: String,
+}
+
+impl PostgresSessionConfig {
+    pub fn new_from_env() -> anyhow::Result<Option<Self>> {
+        // pg not enabled
+        if env::var("PG_ENABLED").is_err() {
+            return Ok(None);
+        }
+
+        let env_pg_config = env::var("PG_CONFIG").context("PG_CONFIG not found")?;
+
+        let ssl_config = if env_pg_config
+            .parse::<tokio_postgres::Config>()?
+            .get_ssl_mode()
+            .eq(&SslMode::Disable)
+        {
+            None
+        } else {
+            let env_ca_pem_b64 = env::var("CA_PEM_B64").context("CA_PEM_B64 not found")?;
+            let env_client_pks_b64 =
+                env::var("CLIENT_PKS_B64").context("CLIENT_PKS_B64 not found")?;
+            let env_client_pks_pass =
+                env::var("CLIENT_PKS_PASS").context("CLIENT_PKS_PASS not found")?;
+
+            Some(PostgresSessionSslConfig {
+                ca_pem_b64: env_ca_pem_b64,
+                client_pks_b64: env_client_pks_b64,
+                client_pks_pass: env_client_pks_pass,
+            })
+        };
+
+        Ok(Some(Self {
+            pg_config: env_pg_config,
+            ssl: ssl_config,
+        }))
+    }
+}

history/src/postgres/postgres_session.rs

@@ -7,6 +7,8 @@ use solana_lite_rpc_core::encoding::BinaryEncoding;
 use tokio::sync::RwLock;
 use tokio_postgres::{config::SslMode, tls::MakeTlsConnect, types::ToSql, Client, NoTls, Socket};
 
+use super::postgres_config::{PostgresSessionConfig, PostgresSessionSslConfig};
+
 const MAX_QUERY_SIZE: usize = 200_000; // 0.2 mb
 
 pub trait SchemaSize {
@@ -36,18 +38,19 @@ pub struct PostgresSession {
 }
 
 impl PostgresSession {
-    pub async fn new() -> anyhow::Result<Self> {
-        let pg_config = std::env::var("PG_CONFIG").context("env PG_CONFIG not found")?;
+    pub async fn new(
+        PostgresSessionConfig { pg_config, ssl }: &PostgresSessionConfig,
+    ) -> anyhow::Result<Self> {
         let pg_config = pg_config.parse::<tokio_postgres::Config>()?;
 
         let client = if let SslMode::Disable = pg_config.get_ssl_mode() {
             Self::spawn_connection(pg_config, NoTls).await?
         } else {
-            let ca_pem_b64 = std::env::var("CA_PEM_B64").context("env CA_PEM_B64 not found")?;
-            let client_pks_b64 =
-                std::env::var("CLIENT_PKS_B64").context("env CLIENT_PKS_B64 not found")?;
-            let client_pks_password =
-                std::env::var("CLIENT_PKS_PASS").context("env CLIENT_PKS_PASS not found")?;
+            let PostgresSessionSslConfig {
+                ca_pem_b64,
+                client_pks_b64,
+                client_pks_pass,
+            } = ssl.as_ref().unwrap();
 
             let ca_pem = BinaryEncoding::Base64
                 .decode(ca_pem_b64)
@@ -58,9 +61,7 @@ impl PostgresSession {
 
             let connector = TlsConnector::builder()
                 .add_root_certificate(Certificate::from_pem(&ca_pem)?)
-                .identity(
-                    Identity::from_pkcs12(&client_pks, &client_pks_password).context("Identity")?,
-                )
+                .identity(Identity::from_pkcs12(&client_pks, client_pks_pass).context("Identity")?)
                 .danger_accept_invalid_hostnames(true)
                 .danger_accept_invalid_certs(true)
                 .build()?;
@@ -136,21 +137,23 @@ impl PostgresSession {
 #[derive(Clone)]
 pub struct PostgresSessionCache {
     session: Arc<RwLock<PostgresSession>>,
+    config: PostgresSessionConfig,
 }
 
 impl PostgresSessionCache {
-    pub async fn new() -> anyhow::Result<Self> {
-        let session = PostgresSession::new().await?;
+    pub async fn new(config: PostgresSessionConfig) -> anyhow::Result<Self> {
+        let session = PostgresSession::new(&config).await?;
         Ok(Self {
             session: Arc::new(RwLock::new(session)),
+            config,
         })
     }
 
     pub async fn get_session(&self) -> anyhow::Result<PostgresSession> {
         let session = self.session.read().await;
         if session.client.is_closed() {
             drop(session);
-            let session = PostgresSession::new().await?;
+            let session = PostgresSession::new(&self.config).await?;
             *self.session.write().await = session.clone();
             Ok(session)
         } else {