Bitcoin Core Implementation

lib/base/README.md

@@ -258,9 +258,9 @@ aws ssm start-session --target $INSTANCE_ID --region $AWS_REGION
 ```bash
 sudo su bcuser
 # Execution client logs:
-docker logs --tail 50 node_execution_1 -f
+docker logs --tail 50 node-execution-1 -f
 # Base client logs:
-docker logs --tail 50 node_node_1 -f
+docker logs --tail 50 node-node-1 -f
 ```
 2. How to check the logs from the EC2 user-data script?
 

lib/base/app.ts

@@ -17,12 +17,13 @@ new BaseCommonStack(app, "base-common", {
 });
 
 new BaseSingleNodeStack(app, "base-single-node", {
-  stackName: `base-single-node-${config.baseNodeConfig.baseNodeConfiguration}-${config.baseNodeConfig.baseNetworkId}`,
+  stackName: `base-single-node-${config.baseNodeConfig.baseClient}-${config.baseNodeConfig.baseNodeConfiguration}-${config.baseNodeConfig.baseNetworkId}`,
   env: { account: config.baseConfig.accountId, region: config.baseConfig.region },
 
   instanceType: config.baseNodeConfig.instanceType,
   instanceCpuType: config.baseNodeConfig.instanceCpuType,
   baseNetworkId: config.baseNodeConfig.baseNetworkId,
+  baseClient: config.baseNodeConfig.baseClient,
   baseNodeConfiguration: config.baseNodeConfig.baseNodeConfiguration,
   restoreFromSnapshot: config.baseNodeConfig.restoreFromSnapshot,
   l1ExecutionEndpoint: config.baseNodeConfig.l1ExecutionEndpoint,
@@ -32,12 +33,13 @@ new BaseSingleNodeStack(app, "base-single-node", {
 });
 
 new BaseHANodesStack(app, "base-ha-nodes", {
-  stackName: `base-ha-nodes-${config.baseNodeConfig.baseNodeConfiguration}-${config.baseNodeConfig.baseNetworkId}`,
+  stackName: `base-ha-nodes-${config.baseNodeConfig.baseClient}-${config.baseNodeConfig.baseNodeConfiguration}-${config.baseNodeConfig.baseNetworkId}`,
   env: { account: config.baseConfig.accountId, region: config.baseConfig.region },
 
   instanceType: config.baseNodeConfig.instanceType,
   instanceCpuType: config.baseNodeConfig.instanceCpuType,
   baseNetworkId: config.baseNodeConfig.baseNetworkId,
+  baseClient: config.baseNodeConfig.baseClient,
   baseNodeConfiguration: config.baseNodeConfig.baseNodeConfiguration,
   restoreFromSnapshot: config.baseNodeConfig.restoreFromSnapshot,
   l1ExecutionEndpoint: config.baseNodeConfig.l1ExecutionEndpoint,

lib/base/lib/assets/instance/cfn-hup/setup.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+if [ -n "$1" ]; then
+  export STACK_ID=$1
+else
+  echo "Error: No Stack ID is provided"
+  echo "Usage: instance/cfn-hup/setup.sh <stack_id> <aws_region>"
+  exit 1
+fi
+
+if [ -n "$2" ]; then
+  export AWS_REGION=$2
+else
+  echo "Error: No AWS Region is provided"
+  echo "Usage: instance/cfn-hup/setup.sh <stack_id> <aws_region>"
+  exit 1
+fi
+
+  echo "Install CloudFormation helper scripts"
+  mkdir -p /opt/aws/
+  pip3 install --break-system-packages https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz
+  ln -s /usr/local/init/ubuntu/cfn-hup /etc/init.d/cfn-hup
+
+  echo "Configuring CloudFormation helper scripts"
+  mkdir -p /etc/cfn/
+  mv /opt/instance/cfn-hup/cfn-hup.conf /etc/cfn/cfn-hup.conf
+  sed -i "s;__AWS_STACK_ID__;\"$STACK_ID\";g" /etc/cfn/cfn-hup.conf
+  sed -i "s;__AWS_REGION__;\"$AWS_REGION\";g" /etc/cfn/cfn-hup.conf
+
+  mkdir -p /etc/cfn/hooks.d/system
+  mv /opt/instance/cfn-hup/cfn-auto-reloader.conf /etc/cfn/hooks.d/cfn-auto-reloader.conf
+  sed -i "s;__AWS_STACK_NAME__;\"$STACK_NAME\";g" /etc/cfn/hooks.d/cfn-auto-reloader.conf
+  sed -i "s;__AWS_REGION__;\"$AWS_REGION\";g" /etc/cfn/hooks.d/cfn-auto-reloader.conf
+
+  echo "Starting CloudFormation helper scripts as a service"
+  mv /opt/instance/cfn-hup/cfn-hup.service  /etc/systemd/system/cfn-hup.service
+
+  systemctl daemon-reload
+  systemctl enable --now cfn-hup
+  systemctl start cfn-hup.service

lib/base/lib/assets/instance/storage/restore-from-snapshot.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+set +e
+
+source /etc/cdk_environment
+
+echo "Downloading Snapshot."
+
+cd /data
+
+SNAPSHOT_FILE_NAME=snapshot.tar.gz
+SNAPSHOT_DIR=/data
+SNAPSHOT_DOWNLOAD_STATUS=-1
+
+if [ "$SNAPSHOT_URL" == "none" ] || [ -z "${SNAPSHOT_URL}" ]; then
+
+  case $BASE_CLIENT in
+    "geth")
+      LATEST_SNAPSHOT_FILE_NAME=$(curl https://$NETWORK_ID-full-snapshots.base.org/latest)
+      SNAPSHOT_URL=https://$NETWORK_ID-full-snapshots.base.org/$LATEST_SNAPSHOT_FILE_NAME
+      ;;
+    "reth")
+      LATEST_SNAPSHOT_FILE_NAME=$(curl https://$NETWORK_ID-reth-archive-snapshots.base.org/latest)
+      SNAPSHOT_URL=https://$NETWORK_ID-reth-archive-snapshots.base.org/$LATEST_SNAPSHOT_FILE_NAME
+      ;;
+    *)
+      # Geth
+      LATEST_SNAPSHOT_FILE_NAME=$(curl https://$NETWORK_ID-full-snapshots.base.org/latest)
+      SNAPSHOT_URL=https://$NETWORK_ID-full-snapshots.base.org/$LATEST_SNAPSHOT_FILE_NAME
+      ;;
+  esac
+fi
+
+while (( SNAPSHOT_DOWNLOAD_STATUS != 0 ))
+do
+        PIDS=$(pgrep wget)
+        if [ -z "$PIDS" ]; then
+            wget $SNAPSHOT_URL -P $SNAPSHOT_DIR -O $SNAPSHOT_FILE_NAME
+        fi
+        SNAPSHOT_DOWNLOAD_STATUS=$?
+        pid=$(pidof wget)
+        wait $pid
+        echo "wget exit."
+        case $SNAPSHOT_DOWNLOAD_STATUS in
+                8)
+                        echo "Server error."
+                        exit 8
+                        ;;
+                3)
+                        echo "No space left on device."
+                        exit 3
+                        ;;
+                *)
+                        continue
+                        ;;
+        esac
+done
+echo "Downloading Snapshot script finished"
+
+sleep 60
+
+echo "Starting snapshot decompression ..."
+
+tar --use-compress-program=unzstd -xvf  $SNAPSHOT_DIR/$SNAPSHOT_FILE_NAME -C /data 2>&1 | tee unzip.log && echo "decompressed successfully..." || echo "decompression failed..." >> snapshots-decompression.log
+
+echo "Decompressed snapshot, cleaning up..."
+
+mv /data/snapshots/$NETWORK_ID/download/* /data && \
+rm -rf /data/snapshots && \
+rm -rf $SNAPSHOT_DIR/$SNAPSHOT_FILE_NAME
+
+echo "Snapshot is ready, starting the service.."
+
+chown -R bcuser:bcuser /data
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now node

lib/base/lib/assets/instance/storage/setup.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+
+make_fs () {
+  # If file system = to ext4 use mkfs.ext4, if xfs use mkfs.xfs
+  if [ -z "$1" ]; then
+    echo "Error: No file system type provided."
+    echo "Usage: make_fs <file system type [ xfs | ext4 ]> <target_volume_id>"
+    exit 1
+  fi
+
+  if [ -z "$2" ]; then
+    echo "Error: No target volume ID provided."
+    echo "Usage: make_fs <file system type [ xfs | ext4 ]> <target_volume_id>"
+    exit 1
+  fi
+
+  local file_system=$1
+  local volume_id=$2
+  if [ "$file_system" == "ext4" ]; then
+    mkfs -t ext4 "$volume_id"
+    return "$?"
+  else
+    mkfs.xfs -f "$volume_id"
+    return "$?"
+  fi
+}
+
+# We need an nvme disk that is not mounted and not partitioned
+get_all_empty_nvme_disks () {
+  local all_not_mounted_nvme_disks
+  local all_mounted_nvme_partitions
+  local unmounted_nvme_disks=()
+  local sorted_unmounted_nvme_disks
+
+  #The disk will only be mounted when the nvme disk is larger than 100GB to avoid storing blockchain node data directly on the root EBS disk (which is 46GB by default)
+  all_not_mounted_nvme_disks=$(lsblk -lnb | awk '{if ($7 == "" && $4 > 100000000) {print $1}}' | grep nvme)
+  all_mounted_nvme_partitions=$(mount | awk '{print $1}' | grep /dev/nvme)
+  for disk in ${all_not_mounted_nvme_disks[*]}; do
+    if [[ ! "${all_mounted_nvme_partitions[*]}" =~ $disk ]]; then
+      unmounted_nvme_disks+=("$disk")
+    fi
+  done
+  # Sort the array
+  sorted_unmounted_nvme_disks=($(printf '%s\n' "${unmounted_nvme_disks[*]}" | sort))
+  echo "${sorted_unmounted_nvme_disks[*]}"
+}
+
+get_next_empty_nvme_disk () {
+  local sorted_unmounted_nvme_disks
+  sorted_unmounted_nvme_disks=($(get_all_empty_nvme_disks))
+  # Return the first unmounted nvme disk
+  echo "/dev/${sorted_unmounted_nvme_disks[0]}"
+}
+
+# Add input as command line parameters for name of the directory to mount
+if [ -n "$1" ]; then
+  DIR_NAME=$1
+else
+  echo "Error: No data file system mount path is provided."
+  echo "Usage: instance/storage/setup.sh <file_system_mount_path> <file_system_type [ xfs | ext4 ]> <target_volume_size_in_bytes> "
+  echo "Default file system type is ext4"
+  echo "If you skip <target_volume_size_in_bytes>, script will try to use the first unformatted volume ID."
+  echo "Usage example: instance/storage/setup.sh /data ext4 300000000000000"
+  exit 1
+fi
+
+# Case input for $2 between ext4 and xfs, use ext4 as default
+case $2 in
+  ext4)
+    echo "File system set to ext4"
+    FILE_SYSTEM="ext4"
+    FS_CONFIG="defaults"
+    ;;
+  xfs)
+    echo "File system set to xfs"
+    FILE_SYSTEM="xfs"
+    FS_CONFIG="noatime,nodiratime,nodiscard" # See more: https://cdrdv2-public.intel.com/686417/rocksdb-benchmark-tuning-guide-on-xeon.pdf
+    ;;
+  *)
+    echo "File system set to ext4"
+    FILE_SYSTEM="ext4"
+    FS_CONFIG="defaults"
+    ;;
+esac
+
+if [ -n "$3" ]; then
+  VOLUME_SIZE=$3
+else
+  echo "The size of volume for $DIR_NAME is not specified. Will try to guess volume ID."
+fi
+
+  echo "Checking if $DIR_NAME is mounted, and dont do anything if it is"
+  if [ $(df --output=target | grep -c "$DIR_NAME") -lt 1 ]; then
+
+    if [ -n "$VOLUME_SIZE" ]; then
+      VOLUME_ID=/dev/$(lsblk -lnb | awk -v VOLUME_SIZE_BYTES="$VOLUME_SIZE" '{if ($4== VOLUME_SIZE_BYTES) {print $1}}')
+      echo "Data volume size defined, use respective volume id: $VOLUME_ID"
+    else
+      VOLUME_ID=$(get_next_empty_nvme_disk)
+      echo "Data volume size undefined, trying volume id: $VOLUME_ID"
+    fi
+
+    make_fs $FILE_SYSTEM "$VOLUME_ID"
+
+    sleep 10
+    VOLUME_UUID=$(lsblk -fn -o UUID  "$VOLUME_ID")
+    VOLUME_FSTAB_CONF="UUID=$VOLUME_UUID $DIR_NAME $FILE_SYSTEM $FS_CONFIG 0 2"
+    echo "VOLUME_ID=$VOLUME_ID"
+    echo "VOLUME_UUID=$VOLUME_UUID"
+    echo "VOLUME_FSTAB_CONF=$VOLUME_FSTAB_CONF"
+
+    # Check if data disc is already in fstab and replace the line if it is with the new disc UUID
+    echo "Checking fstab for volume $DIR_NAME"
+    if [ $(grep -c "$DIR_NAME" /etc/fstab) -gt 0 ]; then
+      SED_REPLACEMENT_STRING="$(grep -n "$DIR_NAME" /etc/fstab | cut -d: -f1)s#.*#$VOLUME_FSTAB_CONF#"
+      # if file exists, delete it
+      if [ -f /etc/fstab.bak ]; then
+        rm /etc/fstab.bak
+      fi
+      cp /etc/fstab /etc/fstab.bak
+      sed -i "$SED_REPLACEMENT_STRING" /etc/fstab
+    else
+      echo "$VOLUME_FSTAB_CONF" | tee -a /etc/fstab
+    fi
+
+    mount -a
+    chown -R bcuser:bcuser "$DIR_NAME"
+  else
+    echo "$DIR_NAME volume is mounted, nothing changed"
+  fi

lib/base/lib/assets/node/node-start.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+source /etc/cdk_environment
+
+export NETWORK_ENV=".env.$NETWORK_ID"
+export CLIENT="$BASE_CLIENT"
+export HOST_DATA_DIR="/data"
+
+echo "Script is starting client $CLIENT on $NETWORK_ENV"
+# Start the node
+cd /home/bcuser/node
+docker compose -f /home/bcuser/node/docker-compose.yml up -d
+
+echo "Started"

lib/base/lib/assets/node/node-stop.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+source /etc/cdk_environment
+
+export NETWORK_ENV=".env.$NETWORK_ID"
+export CLIENT=geth
+
+echo "Script is starting client $CLIENT on $NETWORK_ENV"
+# Stop the node
+cd /home/bcuser/node
+docker compose -f /home/bcuser/node/docker-compose.yml down
+
+echo "Stopped"