From ddea15faa9dc0c21b7fc13b76835d79ac8d12177 Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Fri, 31 Jan 2025 13:19:40 -0500 Subject: [PATCH] feat: add alt-da support --- README.md | 40 ++++++++++ main.star | 2 + network_params.yaml | 19 ++++- src/alt-da/da-server/da_server_launcher.star | 76 +++++++++++++++++++ .../op-batcher/op_batcher_launcher.star | 13 +++- src/cl/hildr/hildr_launcher.star | 6 ++ src/cl/op-node/op_node_launcher.star | 5 ++ src/contracts/contract_deployer.star | 35 ++++++++- src/el_cl_launcher.star | 2 + src/l2.star | 16 ++++ src/package_io/input_parser.star | 60 +++++++++++++++ src/package_io/sanity_check.star | 23 ++++++ src/participant_network.star | 3 + 13 files changed, 297 insertions(+), 3 deletions(-) create mode 100644 src/alt-da/da-server/da_server_launcher.star diff --git a/README.md b/README.md index dc86ff53..8332e422 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,29 @@ optimism_package: # A list of optional extra params that will be passed to the supervisor container for modifying its behaviour extra_params: [] + + # AltDA Deploy Configuration, which is passed to op-deployer. + # + # For simplicity we currently enforce chains to all be altda or all rollups. + # Adding a single altda chain to a cluster essentially makes all chains have altda levels of security. + # + # To setup an altda cluster, make sure to + # 1. Set altda_deploy_config.use_altda to true (and da_commitment_type to KeccakCommitment, see TODO below) + # 2. For each chain, + # - Add "da_server" to the additional_services list if it should use alt-da + # - For altda chains, set da_server_params to use an image and cmd of your choice (one could use da-server, another eigenda-proxy, another celestia proxy, etc). If unset, op's default da-server image will be used. + altda_deploy_config: + use_altda: false + # TODO: Is this field redundant? Afaiu setting it to GenericCommitment will not deploy the + # DAChallengeContract, and hence is equivalent to setting use_altda to false. + # Furthermore, altda rollups using generic commitments might anyways need to support failing over + # to keccak commitments if the altda layer is down. + da_commitment_type: KeccakCommitment + da_challenge_window: 100 + da_resolve_window: 100 + da_bond_size: 0 + da_resolver_refund_percentage: 0 + # An array of L2 networks to run chains: # Specification of the optimism-participants in the network @@ -391,8 +414,25 @@ optimism_package: # Available services: # - blockscout # - rollup-boost + # - da_server additional_services: [] + # Configuration for da-server - https://specs.optimism.io/experimental/alt-da.html#da-server + # TODO: each op-node and op-batcher should potentially have their own da-server, instead of sharing one like we currently do. For eg batcher needs to write via its da-server, whereas op-nodes don't. + da_server_params: + image: us-docker.pkg.dev/oplabs-tools-artifacts/images/da-server:latest + # Command to pass to the container. + # This is kept maximally generic to allow for any possible configuration, given that different + # da layer da-servers might have completely different flags. + # The below arguments are also the default, so can be omitted, and will work as long as the image + # is the da-server above (which is also the default, so can also be omitted). + cmd: + - "da-server" + - "--file.path=/home" + - "--addr=0.0.0.0" + - "--port=3100" + - "--log.level=debug" + # L2 contract deployer configuration - used for all L2 networks # The docker image that should be used for the L2 contract deployer op_contract_deployer_params: diff --git a/main.star b/main.star index 0b241a0d..3e29b215 100644 --- a/main.star +++ b/main.star @@ -44,6 +44,7 @@ def run(plan, args): global_node_selectors = optimism_args_with_right_defaults.global_node_selectors global_log_level = optimism_args_with_right_defaults.global_log_level persistent = optimism_args_with_right_defaults.persistent + altda_deploy_config = optimism_args_with_right_defaults.altda_deploy_config observability_params = optimism_args_with_right_defaults.observability interop_params = optimism_args_with_right_defaults.interop @@ -94,6 +95,7 @@ def run(plan, args): l1_config_env_vars, optimism_args_with_right_defaults, l1_network, + altda_deploy_config, ) jwt_file = plan.upload_files( diff --git a/network_params.yaml b/network_params.yaml index a7f2d9cc..0dfc12b8 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -1,4 +1,11 @@ optimism_package: + altda_deploy_config: + use_altda: true + da_commitment_type: KeccakCommitment + da_challenge_window: 100 + da_resolve_window: 100 + da_bond_size: 0 + da_resolver_refund_percentage: 0 chains: - participants: - el_type: op-geth @@ -39,7 +46,17 @@ optimism_package: mev_params: builder_host: "" builder_port: "" - additional_services: [] + da_server_params: + image: us-docker.pkg.dev/oplabs-tools-artifacts/images/da-server:latest + # A list of optional extra params that will be passed to the da-server container for modifying its behaviour + cmd: + - "da-server" + - "--file.path=/home" + - "--addr=0.0.0.0" + - "--port=3100" + - "--log.level=debug" + additional_services: + - da_server op_contract_deployer_params: image: us-docker.pkg.dev/oplabs-tools-artifacts/images/op-deployer:v0.0.11 l1_artifacts_locator: https://storage.googleapis.com/oplabs-contract-artifacts/artifacts-v1-c193a1863182092bc6cb723e523e8313a0f4b6e9c9636513927f1db74c047c15.tar.gz diff --git a/src/alt-da/da-server/da_server_launcher.star b/src/alt-da/da-server/da_server_launcher.star new file mode 100644 index 00000000..757437b9 --- /dev/null +++ b/src/alt-da/da-server/da_server_launcher.star @@ -0,0 +1,76 @@ +shared_utils = import_module( + "github.com/ethpandaops/ethereum-package/src/shared_utils/shared_utils.star" +) +constants = import_module( + "github.com/ethpandaops/ethereum-package/src/package_io/constants.star" +) + +# Port IDs +DA_SERVER_HTTP_PORT_ID = "http" + +# Port nums +DA_SERVER_HTTP_PORT_NUM = 3100 + + +def get_used_ports(): + used_ports = { + DA_SERVER_HTTP_PORT_ID: shared_utils.new_port_spec( + DA_SERVER_HTTP_PORT_NUM, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + } + return used_ports + + +def launch_da_server( + plan, + service_name, + image, + cmd, +): + config = get_da_server_config( + plan, + service_name, + image, + cmd, + ) + + da_server_service = plan.add_service(service_name, config) + + http_url = "http://{0}:{1}".format( + da_server_service.ip_address, DA_SERVER_HTTP_PORT_NUM + ) + # da_server_context is passed as argument to op-batcher and op-node(s) + return new_da_server_context( + http_url=http_url, + ) + + +def get_da_server_config( + plan, + service_name, + image, + cmd, +): + ports = get_used_ports() + + return ServiceConfig( + image=image, + ports=ports, + cmd=cmd, + private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER, + ) + + +def disabled_da_server_context(): + return new_da_server_context( + http_url="", + ) + + +def new_da_server_context(http_url): + return struct( + enabled=http_url != "", + http_url=http_url, + ) diff --git a/src/batcher/op-batcher/op_batcher_launcher.star b/src/batcher/op-batcher/op_batcher_launcher.star index 6fcecfa3..1d05826f 100644 --- a/src/batcher/op-batcher/op_batcher_launcher.star +++ b/src/batcher/op-batcher/op_batcher_launcher.star @@ -45,6 +45,7 @@ def launch( gs_batcher_private_key, batcher_params, observability_helper, + da_server_context, ): batcher_service_name = "{0}".format(service_name) @@ -58,6 +59,7 @@ def launch( gs_batcher_private_key, batcher_params, observability_helper, + da_server_context, ) batcher_service = plan.add_service(service_name, config) @@ -82,6 +84,7 @@ def get_batcher_config( gs_batcher_private_key, batcher_params, observability_helper, + da_server_context, ): ports = dict(get_used_ports()) @@ -100,7 +103,15 @@ def get_batcher_config( "--max-channel-duration=1", "--l1-eth-rpc=" + l1_config_env_vars["L1_RPC_URL"], "--private-key=" + gs_batcher_private_key, - "--data-availability-type=blobs", + # da commitments currently have to be sent as calldata to the batcher inbox + "--data-availability-type=" + "calldata" + if da_server_context.enabled + else "blobs", + "--altda.enabled=" + str(da_server_context.enabled), + "--altda.da-server=" + da_server_context.http_url, + # This flag is very badly named, but is needed in order to let the da-server compute the commitment. + # This leads to sending POST requests to /put instead of /put/ + "--altda.da-service", ] # apply customizations diff --git a/src/cl/hildr/hildr_launcher.star b/src/cl/hildr/hildr_launcher.star index a01d2e3a..64cd2b90 100644 --- a/src/cl/hildr/hildr_launcher.star +++ b/src/cl/hildr/hildr_launcher.star @@ -78,6 +78,7 @@ def launch( sequencer_enabled, observability_helper, interop_params, + da_server_context, ): # beacon_node_identity_recipe = PostHttpRequestRecipe( # endpoint="/", @@ -109,6 +110,7 @@ def launch( l1_config_env_vars, sequencer_enabled, observability_helper, + da_server_context, ) beacon_service = plan.add_service(service_name, config) @@ -155,6 +157,7 @@ def get_beacon_config( l1_config_env_vars, sequencer_enabled, observability_helper, + da_server_context, ): EXECUTION_ENGINE_ENDPOINT = "http://{0}:{1}".format( el_context.ip_addr, @@ -183,6 +186,9 @@ def get_beacon_config( ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS, launcher.network_params.network_id, ), + # TODO: support altda flags once they are implemented. + # See https://github.com/optimism-java/hildr/issues/134 + # eg: "--altda.enabled=" + str(da_server_context.enabled), ] # configure files diff --git a/src/cl/op-node/op_node_launcher.star b/src/cl/op-node/op_node_launcher.star index 735ee22b..50cc6f6a 100644 --- a/src/cl/op-node/op_node_launcher.star +++ b/src/cl/op-node/op_node_launcher.star @@ -77,6 +77,7 @@ def launch( sequencer_enabled, observability_helper, interop_params, + da_server_context, ): beacon_node_identity_recipe = PostHttpRequestRecipe( endpoint="/", @@ -110,6 +111,7 @@ def launch( sequencer_enabled, observability_helper, interop_params, + da_server_context, ) beacon_service = plan.add_service(service_name, config) @@ -158,6 +160,7 @@ def get_beacon_config( sequencer_enabled, observability_helper, interop_params, + da_server_context, ): ports = dict(get_used_ports(BEACON_DISCOVERY_PORT_NUM)) @@ -190,6 +193,8 @@ def get_beacon_config( "--p2p.listen.tcp={0}".format(BEACON_DISCOVERY_PORT_NUM), "--p2p.listen.udp={0}".format(BEACON_DISCOVERY_PORT_NUM), "--safedb.path={0}".format(BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER), + "--altda.enabled=" + str(da_server_context.enabled), + "--altda.da-server=" + da_server_context.http_url, ] # configure files diff --git a/src/contracts/contract_deployer.star b/src/contracts/contract_deployer.star index d4013a39..ce7bbb77 100644 --- a/src/contracts/contract_deployer.star +++ b/src/contracts/contract_deployer.star @@ -20,7 +20,9 @@ CANNED_VALUES = ( ) -def deploy_contracts(plan, priv_key, l1_config_env_vars, optimism_args, l1_network): +def deploy_contracts( + plan, priv_key, l1_config_env_vars, optimism_args, l1_network, altda_args +): l2_chain_ids_list = [ str(chain.network_params.network_id) for chain in optimism_args.chains ] @@ -201,6 +203,37 @@ def deploy_contracts(plan, priv_key, l1_config_env_vars, optimism_args, l1_netwo address_update( chain_key(i, "roles.unsafeBlockSigner"), "sequencer", chain_id ), + # altda deploy config + ( + "bool", + chain_key(i, "dangerousAltDAConfig.useAltDA"), + altda_args.use_altda, + ), + ( + "string", + chain_key(i, "dangerousAltDAConfig.daCommitmentType"), + altda_args.da_commitment_type, + ), + ( + "int", + chain_key(i, "dangerousAltDAConfig.daChallengeWindow"), + altda_args.da_challenge_window, + ), + ( + "int", + chain_key(i, "dangerousAltDAConfig.daResolveWindow"), + altda_args.da_resolve_window, + ), + ( + "int", + chain_key(i, "dangerousAltDAConfig.daBondSize"), + altda_args.da_bond_size, + ), + ( + "int", + chain_key(i, "dangerousAltDAConfig.daResolverRefundPercentage"), + altda_args.da_resolver_refund_percentage, + ), ] ) intent_updates.extend([(t, chain_key(i, k), v) for t, k, v in CANNED_VALUES]) diff --git a/src/el_cl_launcher.star b/src/el_cl_launcher.star index d6a47709..86ddcf4d 100644 --- a/src/el_cl_launcher.star +++ b/src/el_cl_launcher.star @@ -44,6 +44,7 @@ def launch( additional_services, observability_helper, interop_params, + da_server_context, ): el_launchers = { "op-geth": { @@ -336,6 +337,7 @@ def launch( sequencer_enabled, observability_helper, interop_params, + da_server_context, ) for metrics_info in [x for x in cl_context.cl_nodes_metrics_info if x != None]: diff --git a/src/l2.star b/src/l2.star index 28eeaf6b..ea853f6f 100644 --- a/src/l2.star +++ b/src/l2.star @@ -1,5 +1,6 @@ participant_network = import_module("./participant_network.star") blockscout = import_module("./blockscout/blockscout_launcher.star") +da_server_launcher = import_module("./alt-da/da-server/da_server_launcher.star") contract_deployer = import_module("./contracts/contract_deployer.star") input_parser = import_module("./package_io/input_parser.star") util = import_module("./util.star") @@ -30,6 +31,20 @@ def launch_l2( plan.print("Deploying L2 with name {0}".format(network_params.name)) + # we need to launch da-server before launching the participant network + # because op-batcher and op-node(s) need to know the da-server url, if present + da_server_context = da_server_launcher.disabled_da_server_context() + if "da_server" in l2_args.additional_services: + da_server_image = l2_args.da_server_params.image + plan.print("Launching da-server") + da_server_context = da_server_launcher.launch_da_server( + plan, + "da-server-{0}".format(l2_services_suffix), + da_server_image, + l2_args.da_server_params.cmd, + ) + plan.print("Successfully launched da-server") + all_l2_participants = participant_network.launch_participant_network( plan, l2_args.participants, @@ -50,6 +65,7 @@ def launch_l2( l2_args.additional_services, observability_helper, interop_params, + da_server_context, ) all_el_contexts = [] diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 62134775..88dd3b2d 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -37,6 +37,22 @@ DEFAULT_SIDECAR_IMAGES = { "rollup-boost": "flashbots/rollup-boost:latest", } +DEFAULT_DA_SERVER_PARAMS = { + "image": "us-docker.pkg.dev/oplabs-tools-artifacts/images/da-server:latest", + "cmd": [ + "da-server", # uses keccak commitments by default + # We use the file storage backend instead of s3 for simplicity. + # Blobs and commitments are stored in the /home directory (which already exists). + # Note that this storage is ephemeral because we aren't mounting an external kurtosis file. + # This means that the data is lost when the container is deleted. + "--file.path=/home", + "--addr=0.0.0.0", + "--port=3100", + "--log.level=debug", + ], +} + + DEFAULT_ADDITIONAL_SERVICES = [] @@ -98,6 +114,16 @@ def input_parser(plan, input_args): extra_params=results["interop"]["supervisor_params"]["extra_params"], ), ), + altda_deploy_config=struct( + use_altda=results["altda_deploy_config"]["use_altda"], + da_commitment_type=results["altda_deploy_config"]["da_commitment_type"], + da_challenge_window=results["altda_deploy_config"]["da_challenge_window"], + da_resolve_window=results["altda_deploy_config"]["da_resolve_window"], + da_bond_size=results["altda_deploy_config"]["da_bond_size"], + da_resolver_refund_percentage=results["altda_deploy_config"][ + "da_resolver_refund_percentage" + ], + ), chains=[ struct( participants=[ @@ -199,6 +225,11 @@ def input_parser(plan, input_args): builder_host=result["mev_params"]["builder_host"], builder_port=result["mev_params"]["builder_port"], ), + da_server_params=struct( + enabled=result["da_server_params"]["enabled"], + image=result["da_server_params"]["image"], + cmd=result["da_server_params"]["cmd"], + ), additional_services=result["additional_services"], ) for result in results["chains"] @@ -250,6 +281,11 @@ def parse_network_params(plan, input_args): input_args.get("interop", {}).get("supervisor_params", {}) ) + # configure altda + + results["altda_deploy_config"] = default_altda_deploy_config() + results["altda_deploy_config"].update(input_args.get("altda_deploy_config", {})) + # configure chains chains = [] @@ -271,6 +307,8 @@ def parse_network_params(plan, input_args): mev_params = default_mev_params() mev_params.update(chain.get("mev_params", {})) + da_server_params = default_da_server_params() + da_server_params.update(chain.get("da_server_params", {})) network_name = network_params["name"] network_id = network_params["network_id"] @@ -347,6 +385,7 @@ def parse_network_params(plan, input_args): "challenger_params": challenger_params, "proposer_params": proposer_params, "mev_params": mev_params, + "da_server_params": da_server_params, "additional_services": chain.get( "additional_services", DEFAULT_ADDITIONAL_SERVICES ), @@ -371,6 +410,7 @@ def default_optimism_params(): return { "observability": default_observability_params(), "interop": default_interop_params(), + "altda": default_altda_deploy_config(), "chains": default_chains(), "op_contract_deployer_params": default_op_contract_deployer_params(), "global_log_level": "info", @@ -415,6 +455,17 @@ def default_interop_params(): } +def default_altda_deploy_config(): + return { + "use_altda": "false", + "da_commitment_type": "KeccakCommitment", + "da_challenge_window": 100, + "da_resolve_window": 100, + "da_bond_size": 0, + "da_resolver_refund_percentage": 0, + } + + def default_supervisor_params(): return { "image": DEFAULT_SUPERVISOR_IMAGES["op-supervisor"], @@ -440,6 +491,7 @@ def default_chains(): "challenger_params": default_challenger_params(), "proposer_params": default_proposer_params(), "mev_params": default_mev_params(), + "da_server_params": default_da_server_params(), "additional_services": DEFAULT_ADDITIONAL_SERVICES, } ] @@ -574,3 +626,11 @@ def default_ethereum_package_network_params(): ), } } + + +def default_da_server_params(): + return { + "enabled": False, + "image": DEFAULT_DA_SERVER_PARAMS["image"], + "cmd": DEFAULT_DA_SERVER_PARAMS["cmd"], + } diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index 5d4835f6..f99a0e3d 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -34,6 +34,15 @@ SUPERVISOR_PARAMS = [ "extra_params", ] +ALTDA_DEPLOY_CONFIG_PARAMS = [ + "use_altda", + "da_commitment_type", + "da_challenge_window", + "da_resolve_window", + "da_bond_size", + "da_resolver_refund_percentage", +] + PARTICIPANT_CATEGORIES = { "participants": [ "el_type", @@ -92,6 +101,10 @@ SUBCATEGORY_PARAMS = { ], "proposer_params": ["image", "extra_params", "game_type", "proposal_interval"], "mev_params": ["rollup_boost_image", "builder_host", "builder_port"], + "da_server_params": [ + "image", + "cmd", + ], } OP_CONTRACT_DEPLOYER_PARAMS = [ @@ -106,11 +119,13 @@ OP_CONTRACT_DEPLOYER_GLOBAL_DEPLOY_OVERRIDES = ["faultGameAbsolutePrestate"] ADDITIONAL_SERVICES_PARAMS = [ "blockscout", "rollup-boost", + "da_server", ] ROOT_PARAMS = [ "observability", "interop", + "altda_deploy_config", "chains", "op_contract_deployer_params", "global_log_level", @@ -200,6 +215,14 @@ def sanity_check(plan, optimism_config): SUPERVISOR_PARAMS, ) + if "altda_deploy_config" in optimism_config: + validate_params( + plan, + optimism_config["altda_deploy_config"], + "altda_deploy_config", + ALTDA_DEPLOY_CONFIG_PARAMS, + ) + chains = optimism_config.get("chains", []) if type(chains) != "list": diff --git a/src/participant_network.star b/src/participant_network.star index c4ebac4f..9fd0fbfa 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -29,6 +29,7 @@ def launch_participant_network( additional_services, observability_helper, interop_params, + da_server_context, ): num_participants = len(participants) # First EL and sequencer CL @@ -49,6 +50,7 @@ def launch_participant_network( additional_services, observability_helper, interop_params, + da_server_context, ) all_participants = [] @@ -89,6 +91,7 @@ def launch_participant_network( batcher_key, batcher_params, observability_helper, + da_server_context, ) game_factory_address = util.read_network_config_value(