From 571cf411c07ced098ed73fcece635c321447add3 Mon Sep 17 00:00:00 2001 From: Godmode Galactus Date: Tue, 5 Nov 2024 16:17:32 +0100 Subject: [PATCH 1/5] adding fixes to run autobahn on eclipse --- lib/dex-orca/src/orca.rs | 15 +++++-- lib/dex-orca/src/orca_dex.rs | 14 +++++-- lib/dex-orca/tests/test_cropper.rs | 10 +++++ .../src/get_program_account.rs | 41 +++++++++++++++++++ lib/router-feed-lib/src/router_rpc_wrapper.rs | 26 +++++++++--- lib/router-lib/src/dex/interface.rs | 1 - .../src/test_tools/generate_dex_rpc_dump.rs | 2 +- scripts/smoke-test.sh | 4 +- 8 files changed, 97 insertions(+), 16 deletions(-) diff --git a/lib/dex-orca/src/orca.rs b/lib/dex-orca/src/orca.rs index d489bdc..c415644 100644 --- a/lib/dex-orca/src/orca.rs +++ b/lib/dex-orca/src/orca.rs @@ -310,10 +310,17 @@ pub async fn fetch_all_whirlpools( .await?; let result = whirlpools .iter() - .map(|account| { - let whirlpool: Whirlpool = - AnchorDeserialize::deserialize(&mut &account.data[8..]).unwrap(); - (account.pubkey, whirlpool) + .filter_map(|account| { + let pubkey = account.pubkey; + let whirlpool: Result = + AnchorDeserialize::deserialize(&mut &account.data[8..]); + match whirlpool { + Ok(whirlpool) => Some((account.pubkey, whirlpool)), + Err(e) => { + error!("Error deserializing whirlpool account : {pubkey:?} error: {e:?}"); + None + } + } }) .collect_vec(); Ok(result) diff --git a/lib/dex-orca/src/orca_dex.rs b/lib/dex-orca/src/orca_dex.rs index 7412879..2e725b6 100644 --- a/lib/dex-orca/src/orca_dex.rs +++ b/lib/dex-orca/src/orca_dex.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use anchor_lang::Id; use anchor_spl::token::spl_token; -use anchor_spl::token::spl_token::state::AccountState; +use anchor_spl::token::spl_token::state::{Account, AccountState}; use anchor_spl::token_2022::Token2022; use anyhow::Context; use itertools::Itertools; @@ -229,7 +229,12 @@ impl OrcaDex { .iter() .filter(|x| { x.1.owner == Token2022::id() - || spl_token::state::Account::unpack(x.1.data()).unwrap().state + || spl_token::state::Account::unpack(x.1.data()) + .unwrap_or(Account { + state: AccountState::Frozen, + ..Default::default() + }) + .state == AccountState::Frozen }) .map(|x| x.0) @@ -246,7 +251,10 @@ impl OrcaDex { // TODO: actually need to dynamically adjust subscriptions based on the tick? let tick_arrays = filtered_pools .iter() - .map(|(pk, wp)| whirlpool_tick_array_pks(wp, pk, program_id)) + .map(|(pk, wp)| { + println!("whirlpool : {pk}, {}", wp.tick_spacing); + whirlpool_tick_array_pks(wp, pk, program_id) + }) .collect_vec(); let edge_pairs = filtered_pools diff --git a/lib/dex-orca/tests/test_cropper.rs b/lib/dex-orca/tests/test_cropper.rs index b46354d..51939cc 100644 --- a/lib/dex-orca/tests/test_cropper.rs +++ b/lib/dex-orca/tests/test_cropper.rs @@ -8,6 +8,16 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc}; #[tokio::test] async fn test_dump_input_data_cropper() -> anyhow::Result<()> { + let is_eclipse = std::env::var("ECLIPSE") + .map(|x| { + let value: bool = x.parse().unwrap(); + value + }) + .unwrap_or_default(); + if is_eclipse { + // crooper is not yet on eclipse + return Ok(()); + } let options = HashMap::from([ ( "program_id".to_string(), diff --git a/lib/router-feed-lib/src/get_program_account.rs b/lib/router-feed-lib/src/get_program_account.rs index 2c87b37..58c68ea 100644 --- a/lib/router-feed-lib/src/get_program_account.rs +++ b/lib/router-feed-lib/src/get_program_account.rs @@ -205,6 +205,47 @@ pub async fn get_compressed_program_account_rpc( Ok((min_slot, snap_result)) } +// called on startup to get the required accounts, few calls with some 100 thousand accounts +#[tracing::instrument(skip_all, level = "trace")] +pub async fn get_uncompressed_program_account_rpc( + rpc_client: &RpcClient, + filters: &HashSet, + config: RpcProgramAccountsConfig, +) -> anyhow::Result<(u64, Vec)> { + let slot = rpc_client.get_slot().await?; + let config = RpcProgramAccountsConfig { + with_context: Some(true), + account_config: RpcAccountInfoConfig { + encoding: Some(UiAccountEncoding::Base64), + min_context_slot: Some(slot), + commitment: config.account_config.commitment, + data_slice: config.account_config.data_slice, + }, + filters: config.filters, + }; + + let mut snap_result = vec![]; + let mut min_slot = u64::MAX; + + // use getGPA compressed if available + for program_id in filters.iter() { + info!("gPA for {}", program_id); + min_slot = slot.min(min_slot); + let account_snapshot = rpc_client + .get_program_accounts_with_config(&program_id, config.clone()) + .await + .map_err_anyhow()?; + tracing::log::debug!("gpa snapshot received {}", program_id); + + let iter = account_snapshot.iter().map(|(pk, account)| { + account_write_from(*pk, slot, SNAP_ACCOUNT_WRITE_VERSION, account.clone()) + }); + snap_result.extend(iter); + } + + Ok((min_slot, snap_result)) +} + // called on startup to get the required accounts, few calls with some 100 thousand accounts #[tracing::instrument(skip_all, level = "trace")] pub async fn get_uncompressed_program_account( diff --git a/lib/router-feed-lib/src/router_rpc_wrapper.rs b/lib/router-feed-lib/src/router_rpc_wrapper.rs index a08369c..9177417 100644 --- a/lib/router-feed-lib/src/router_rpc_wrapper.rs +++ b/lib/router-feed-lib/src/router_rpc_wrapper.rs @@ -9,7 +9,10 @@ use solana_sdk::account::Account; use solana_sdk::pubkey::Pubkey; use crate::account_write::AccountWrite; -use crate::get_program_account::{fetch_multiple_accounts, get_compressed_program_account_rpc}; +use crate::get_program_account::{ + fetch_multiple_accounts, get_compressed_program_account_rpc, + get_uncompressed_program_account_rpc, +}; use crate::router_rpc_client::RouterRpcClientTrait; pub struct RouterRpcWrapper { @@ -52,10 +55,21 @@ impl RouterRpcClientTrait for RouterRpcWrapper { pubkey: &Pubkey, config: RpcProgramAccountsConfig, ) -> anyhow::Result> { - Ok( - get_compressed_program_account_rpc(&self.rpc, &HashSet::from([*pubkey]), config) - .await? - .1, - ) + let disable_compressed = std::env::var::("DISABLE_COMRPESSED_GPA".to_string()) + .unwrap_or("false".to_string()); + let disable_compressed: bool = disable_compressed.trim().parse().unwrap(); + if disable_compressed { + Ok( + get_uncompressed_program_account_rpc(&self.rpc, &HashSet::from([*pubkey]), config) + .await? + .1, + ) + } else { + Ok( + get_compressed_program_account_rpc(&self.rpc, &HashSet::from([*pubkey]), config) + .await? + .1, + ) + } } } diff --git a/lib/router-lib/src/dex/interface.rs b/lib/router-lib/src/dex/interface.rs index 09cd8bd..d79a560 100644 --- a/lib/router-lib/src/dex/interface.rs +++ b/lib/router-lib/src/dex/interface.rs @@ -194,7 +194,6 @@ pub trait DexInterface: Sync + Send { /// simulation tests. fn program_ids(&self) -> HashSet; - /// Initializes an Edge from ChainData (production) or BanksClient (test). /// The Edge will be dropped once a new Edge for the same EdgeIndentifier /// has been initialized. After calling initialize the DexInterface needs diff --git a/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs b/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs index 1133986..8bd587d 100644 --- a/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs +++ b/lib/router-lib/src/test_tools/generate_dex_rpc_dump.rs @@ -227,7 +227,7 @@ pub async fn run_dump_swap_ix_with_custom_amount( continue; }; - debug!( + println!( "#{} || quote: {} => {} : {} => {}", success, id.input_mint(), diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index d63da2b..71a1c7e 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -8,7 +8,9 @@ export DUMP_MAINNET_DATA=1 RUST_LOG=info # define in addition # RPC_HTTP_URL="http://fcs-ams1._peer.internal:18899" - +# for eclipse +# export ECLIPSE=true +# export DISABLE_COMRPESSED_GPA=true # saber DUMP_SABER_START=$(date) From dc3c94244fe42b7b8ada3d51599b2c72779f9f84 Mon Sep 17 00:00:00 2001 From: Godmode Galactus Date: Tue, 5 Nov 2024 17:11:20 +0100 Subject: [PATCH 2/5] adding configuration files for eclipse --- bin/autobahn-router/eclipse/Dockerfile | 25 +++++ .../src/template-config-eclipse.toml | 98 +++++++++++++++++++ bin/autobahn-router/template-config.toml | 2 +- fly-eclipse.toml | 22 +++++ fly.toml | 2 +- 5 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 bin/autobahn-router/eclipse/Dockerfile create mode 100644 bin/autobahn-router/src/template-config-eclipse.toml create mode 100644 fly-eclipse.toml diff --git a/bin/autobahn-router/eclipse/Dockerfile b/bin/autobahn-router/eclipse/Dockerfile new file mode 100644 index 0000000..e95f5ee --- /dev/null +++ b/bin/autobahn-router/eclipse/Dockerfile @@ -0,0 +1,25 @@ +FROM rust:1.76.0 as base +RUN cargo install cargo-chef@0.1.62 --locked +RUN rustup component add rustfmt +RUN apt-get update && apt-get install -y clang cmake ssh +WORKDIR /app + +FROM base AS plan +COPY . . +WORKDIR /app +RUN cargo chef prepare --recipe-path recipe.json + +FROM base as build +COPY --from=plan /app/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json +COPY . . +RUN cargo build --release --bin autobahn-router + +FROM debian:bookworm-slim as run +RUN apt-get update && apt-get -y install ca-certificates libc6 libssl3 libssl-dev openssl + +COPY --from=build /app/target/release/autobahn-router /usr/local/bin/ +COPY --from=build /app/bin/autobahn-router/template-config-eclipse.toml /usr/local/bin/template-config.toml + +RUN adduser --system --group --no-create-home mangouser +USER mangouser diff --git a/bin/autobahn-router/src/template-config-eclipse.toml b/bin/autobahn-router/src/template-config-eclipse.toml new file mode 100644 index 0000000..90937ff --- /dev/null +++ b/bin/autobahn-router/src/template-config-eclipse.toml @@ -0,0 +1,98 @@ +snapshot_timeout_in_seconds = 900 + +[infinity] +enabled = false + +[orca] +enabled = true +mints = [] +take_all_mints = true +add_mango_tokens = false + +[cropper] +enabled = false +mints = [] +take_all_mints = true +add_mango_tokens = false + +[openbook_v2] +enabled = false +mints = [] +take_all_mints = true +add_mango_tokens = false + +[raydium] +enabled = false +mints = [] +take_all_mints = true +add_mango_tokens = false + +[raydium_cp] +enabled = false +mints = [] +take_all_mints = true +add_mango_tokens = false + +[saber] +enabled = false +mints = [] +take_all_mints = true +add_mango_tokens = false + + +[routing] +path_cache_validity_ms = 30000 +path_warming_mode = "ConfiguredMints" +#path_warming_mode = "HotMints" +path_warming_amounts = [100, 1000, 10_000] +path_warming_for_mints = [ + "So11111111111111111111111111111111111111112", # SOL +] +path_warming_interval_secs = 5 +path_warming_max_accounts = [20, 30, 40, 64] +lookup_tables = [] +cooldown_duration_multihop_secs = 30 +cooldown_duration_singlehop_secs = 60 +max_path_length = 3 +retain_path_count = 5 +max_edge_per_pair = 5 +max_edge_per_cold_pair = 2 + +[server] +address = "[::]:8888" + +[metrics] +output_http = true +prometheus_address = "[::]:9091" +output_stdout = false + +[[sources]] +dedup_queue_size = 50000 +rpc_http_url = "$RPC_HTTP_URL" +rpc_support_compression = false +re_snapshot_interval_secs = 1200 +request_timeout_in_seconds = 300 + +[[sources.grpc_sources]] +name = "router-eclipse" +connection_string = "$RPC_HTTP_URL_WITHOUT_TOKEN" +token = "$AMS_RPC_TOKEN" +retry_connection_sleep_secs = 30 + +[price_feed] +birdeye_token = "$BIRDEYE_TOKEN" +refresh_interval_secs = 1200 # every 20 min + +[safety_checks] +check_quote_out_amount_deviation = true +min_quote_out_to_in_amount_ratio = 0.65 + +[hot_mints] +always_hot_mints = [ + "So11111111111111111111111111111111111111112", # SOL +] +keep_latest_count = 50 + +[debug_config] +reprice_using_live_rpc = true +reprice_probability = 0.05 diff --git a/bin/autobahn-router/template-config.toml b/bin/autobahn-router/template-config.toml index 1a521c4..a51ccc7 100644 --- a/bin/autobahn-router/template-config.toml +++ b/bin/autobahn-router/template-config.toml @@ -52,7 +52,7 @@ path_warming_for_mints = [ # "27G8MtK7VtTcCHkpASjSDdkWWYfoqT6ggEuKidVJidD4" # JLP ] path_warming_interval_secs = 5 -path_warming_max_accounts = [20, 25, 30, 35, 40, 64] +path_warming_max_accounts = [20, 30, 40, 64] lookup_tables = ["87TgskchTNEv1uXkGQk1U4zt65tjqbfGAZWNMGAcRRPx", "AgCBUZ6UMWqPLftTxeAqpQxtrfiCyL2HgRfmmM6QTfCj", "A1v3qxN7HbUvtyPnnaoCrKonXjkFLaDHXk3S6R2QfEaw", diff --git a/fly-eclipse.toml b/fly-eclipse.toml new file mode 100644 index 0000000..8abea39 --- /dev/null +++ b/fly-eclipse.toml @@ -0,0 +1,22 @@ +app = "router-eclipse" +primary_region = "fra" +kill_signal = "SIGTERM" +kill_timeout = "30s" + +[build] + dockerfile = 'bin/autobahn-router/eclipse/Dockerfile' + +[experimental] + cmd = ["autobahn-router", "/usr/local/bin/template-config-eclipse.toml"] + +[[vm]] + size = "shared-4x" + memory = "8gb" + +[[restart]] + policy = "always" + retries = 10 + +[metrics] + port = 9091 + path = "/metrics" \ No newline at end of file diff --git a/fly.toml b/fly.toml index 0927c77..77b8498 100644 --- a/fly.toml +++ b/fly.toml @@ -1,5 +1,5 @@ app = "router-2" -primary_region = "dfw" +primary_region = "ams" kill_signal = "SIGTERM" kill_timeout = "30s" From e8f7123beef3534d92e799767994fc46f07a5fc8 Mon Sep 17 00:00:00 2001 From: Riordan Panayides Date: Tue, 5 Nov 2024 17:14:25 +0000 Subject: [PATCH 3/5] add CONFIG_PATH build arg --- bin/autobahn-router/Dockerfile | 3 ++- bin/autobahn-router/eclipse/Dockerfile | 25 ------------------- .../{src => }/template-config-eclipse.toml | 0 fly-eclipse.toml | 9 ++++--- 4 files changed, 8 insertions(+), 29 deletions(-) delete mode 100644 bin/autobahn-router/eclipse/Dockerfile rename bin/autobahn-router/{src => }/template-config-eclipse.toml (100%) diff --git a/bin/autobahn-router/Dockerfile b/bin/autobahn-router/Dockerfile index b671b3b..2469708 100644 --- a/bin/autobahn-router/Dockerfile +++ b/bin/autobahn-router/Dockerfile @@ -16,10 +16,11 @@ COPY . . RUN cargo build --release --bin autobahn-router FROM debian:bookworm-slim as run +ARG CONFIG_PATH=/app/bin/autobahn-router/template-config.toml RUN apt-get update && apt-get -y install ca-certificates libc6 libssl3 libssl-dev openssl COPY --from=build /app/target/release/autobahn-router /usr/local/bin/ -COPY --from=build /app/bin/autobahn-router/template-config.toml /usr/local/bin/template-config.toml +COPY --from=build $CONFIG_PATH /usr/local/bin/template-config.toml RUN adduser --system --group --no-create-home mangouser USER mangouser diff --git a/bin/autobahn-router/eclipse/Dockerfile b/bin/autobahn-router/eclipse/Dockerfile deleted file mode 100644 index e95f5ee..0000000 --- a/bin/autobahn-router/eclipse/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM rust:1.76.0 as base -RUN cargo install cargo-chef@0.1.62 --locked -RUN rustup component add rustfmt -RUN apt-get update && apt-get install -y clang cmake ssh -WORKDIR /app - -FROM base AS plan -COPY . . -WORKDIR /app -RUN cargo chef prepare --recipe-path recipe.json - -FROM base as build -COPY --from=plan /app/recipe.json recipe.json -RUN cargo chef cook --release --recipe-path recipe.json -COPY . . -RUN cargo build --release --bin autobahn-router - -FROM debian:bookworm-slim as run -RUN apt-get update && apt-get -y install ca-certificates libc6 libssl3 libssl-dev openssl - -COPY --from=build /app/target/release/autobahn-router /usr/local/bin/ -COPY --from=build /app/bin/autobahn-router/template-config-eclipse.toml /usr/local/bin/template-config.toml - -RUN adduser --system --group --no-create-home mangouser -USER mangouser diff --git a/bin/autobahn-router/src/template-config-eclipse.toml b/bin/autobahn-router/template-config-eclipse.toml similarity index 100% rename from bin/autobahn-router/src/template-config-eclipse.toml rename to bin/autobahn-router/template-config-eclipse.toml diff --git a/fly-eclipse.toml b/fly-eclipse.toml index 8abea39..765febc 100644 --- a/fly-eclipse.toml +++ b/fly-eclipse.toml @@ -4,13 +4,16 @@ kill_signal = "SIGTERM" kill_timeout = "30s" [build] - dockerfile = 'bin/autobahn-router/eclipse/Dockerfile' + dockerfile = 'bin/autobahn-router/Dockerfile' + +[build.args] + CONFIG_PATH="/app/bin/autobahn-router/template-config-eclipse.toml" [experimental] - cmd = ["autobahn-router", "/usr/local/bin/template-config-eclipse.toml"] + cmd = ["autobahn-router", "/usr/local/bin/template-config.toml"] [[vm]] - size = "shared-4x" + size = "shared-cpu-4x" memory = "8gb" [[restart]] From 4a7d0924b130a04f8e03b3accc89070fe1407c1a Mon Sep 17 00:00:00 2001 From: Godmode Galactus Date: Tue, 5 Nov 2024 18:25:02 +0100 Subject: [PATCH 4/5] minor config changes and bugfixes --- bin/autobahn-router/template-config-eclipse.toml | 2 +- lib/dex-orca/src/orca_dex.rs | 1 - lib/router-feed-lib/src/get_program_account.rs | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/autobahn-router/template-config-eclipse.toml b/bin/autobahn-router/template-config-eclipse.toml index 90937ff..01c59a9 100644 --- a/bin/autobahn-router/template-config-eclipse.toml +++ b/bin/autobahn-router/template-config-eclipse.toml @@ -76,7 +76,7 @@ request_timeout_in_seconds = 300 [[sources.grpc_sources]] name = "router-eclipse" connection_string = "$RPC_HTTP_URL_WITHOUT_TOKEN" -token = "$AMS_RPC_TOKEN" +token = "$RPC_TOKEN" retry_connection_sleep_secs = 30 [price_feed] diff --git a/lib/dex-orca/src/orca_dex.rs b/lib/dex-orca/src/orca_dex.rs index 2e725b6..85023e4 100644 --- a/lib/dex-orca/src/orca_dex.rs +++ b/lib/dex-orca/src/orca_dex.rs @@ -252,7 +252,6 @@ impl OrcaDex { let tick_arrays = filtered_pools .iter() .map(|(pk, wp)| { - println!("whirlpool : {pk}, {}", wp.tick_spacing); whirlpool_tick_array_pks(wp, pk, program_id) }) .collect_vec(); diff --git a/lib/router-feed-lib/src/get_program_account.rs b/lib/router-feed-lib/src/get_program_account.rs index 58c68ea..5b9ae2b 100644 --- a/lib/router-feed-lib/src/get_program_account.rs +++ b/lib/router-feed-lib/src/get_program_account.rs @@ -217,7 +217,7 @@ pub async fn get_uncompressed_program_account_rpc( with_context: Some(true), account_config: RpcAccountInfoConfig { encoding: Some(UiAccountEncoding::Base64), - min_context_slot: Some(slot), + min_context_slot: None, commitment: config.account_config.commitment, data_slice: config.account_config.data_slice, }, From 062354fbbb4c90791131f8a76eba85f75d63b78c Mon Sep 17 00:00:00 2001 From: Godmode Galactus Date: Wed, 6 Nov 2024 13:56:23 +0100 Subject: [PATCH 5/5] increase acceptable slot lag, because eclipse is not so much active yet --- bin/autobahn-router/template-config-eclipse.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/autobahn-router/template-config-eclipse.toml b/bin/autobahn-router/template-config-eclipse.toml index 01c59a9..25bd6e5 100644 --- a/bin/autobahn-router/template-config-eclipse.toml +++ b/bin/autobahn-router/template-config-eclipse.toml @@ -57,6 +57,7 @@ max_path_length = 3 retain_path_count = 5 max_edge_per_pair = 5 max_edge_per_cold_pair = 2 +slot_excessive_lag_max_duration_secs = 1600 [server] address = "[::]:8888"