Merge pull request #1210 from input-output-hk/chore/merge-master-to-conway-era

Merge `master` to `conway-era`

Author: michalrus

compose/smash/Dockerfile

@@ -3,12 +3,22 @@ ARG UBUNTU_VERSION=20.04
 FROM ubuntu:${UBUNTU_VERSION}
 
 ARG DBSYNC_BINARY_URL="https://update-cardano-mainnet.iohk.io/cardano-db-sync/cardano-db-sync-${CARDANO_DB_SYNC_VERSION:-13.1.0.0}-linux.tar.gz"
+ARG DBSYNC_BINARY_URL_ARM64="https://github.com/input-output-hk/ogmios-tracker/releases/download/0.1.0/cardano-db-sync-13.1.0.0-aarch64-linux.tar.gz"
 ENV NETWORK=${NETWORK:-"mainnet"}
 
 RUN apt-get update && apt-get install curl -y
-RUN curl ${DBSYNC_BINARY_URL} --output /tmp/dbsync-binary.tar.gz &&\
-  tar --extract --file /tmp/dbsync-binary.tar.gz --directory /bin ./cardano-smash-server &&\
-  rm /tmp/dbsync-binary.tar.gz
+
+RUN set -x ; if [ "$(uname -m)" = "aarch64" ] ; then \
+      curl -sfL "${DBSYNC_BINARY_URL_ARM64}" --output /tmp/dbsync-binary.tar.gz &&\
+      mkdir -p /opt/cardano-db-sync &&\
+      tar -xzf /tmp/dbsync-binary.tar.gz -C /opt/cardano-db-sync &&\
+      rm /tmp/dbsync-binary.tar.gz &&\
+      ln -sf /opt/cardano-db-sync/bin/* /bin/ ;\
+    else \
+      curl -sfL "${DBSYNC_BINARY_URL}" --output /tmp/dbsync-binary.tar.gz &&\
+      tar --extract --file /tmp/dbsync-binary.tar.gz --directory /bin ./cardano-smash-server &&\
+      rm /tmp/dbsync-binary.tar.gz ;\
+    fi
 
 COPY packages/cardano-services/config/network/${NETWORK}/cardano-db-sync /config/
 

compose/smash/init.sh

@@ -6,16 +6,9 @@ USER=$(cat /run/secrets/postgres_user)
 
 echo "postgres:5432:${DB}:${USER}:${PASSWORD}" >/config/pgpass
 
-_term() {
-  kill $CHILD
-}
+export PGPASSFILE=/config/pgpass
 
-trap _term SIGTERM
-
-PGPASSFILE=/config/pgpass cardano-smash-server \
+exec cardano-smash-server \
   --config /config/config.json \
   --port 3100 \
-  --admins /config/smash-admins.txt &
-
-CHILD=$!
-wait "$CHILD"
+  --admins /config/smash-admins.txt

packages/cardano-services/package.json

@@ -29,7 +29,7 @@
     "cleanup:nm": "rm -rf node_modules",
     "cleanup": "rm -rf dist node_modules",
     "cli": "ts-node --transpile-only src/cli.ts",
-    "compose:up": "docker compose --env-file environments/.env.$NETWORK -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml ${FILES:-} up",
+    "compose:up": "__FIX_UMASK__=$(chmod -R a+r ../../compose/placeholder-secrets) docker compose --env-file environments/.env.$NETWORK -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml ${FILES:-} up",
     "compose:down": "docker compose -p cardano-services-$NETWORK -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/pg-agent.yml -f ../../compose/$(uname -m).yml down -t 120",
     "coverage": "yarn test --coverage || true",
     "circular-deps:check": "madge --circular dist/cjs",

packages/e2e/.gitignore

@@ -9,3 +9,5 @@ node_modules
 dist
 coverage
 .env
+
+*-old-state.*/

packages/e2e/docker-compose.yml

@@ -22,7 +22,8 @@ services:
     ports:
       - 3001:3001
     volumes:
-      - ./local-network/network-files/node-sp1/:/root/network-files/node-sp1
+      # This breaks running in Docker Desktop for macOS, is it *really* needed? I see no usage of cardano-cli in the repo
+      #- ./local-network/network-files/node-sp1/:/root/network-files/node-sp1
       - ./local-network/config:/root/config
       - sdk-ipc:/sdk-ipc
 
@@ -37,7 +38,9 @@ services:
       NGINX_PORT: 80
     healthcheck:
       test: ['CMD-SHELL', 'wget -O /dev/null http://localhost || exit 1']
-      timeout: 10s
+      start_period: 5s
+      interval: 5s
+      timeout: 2s
 
   cardano-node:
     depends_on:
@@ -65,6 +68,9 @@ services:
       - ./local-network/config/network:/config
 
   cardano-submit-api:
+    depends_on:
+      local-testnet:
+        condition: service_healthy
     volumes:
       - ./local-network/config/network:/config
 

packages/e2e/local-network/.dockerignore

@@ -0,0 +1 @@
+bin/

packages/e2e/local-network/Dockerfile

@@ -6,27 +6,39 @@ ENV DEBIAN_FRONTEND=nonintercative
 
 WORKDIR /build
 ARG CARDANO_NODE_BUILD_URL=https://github.com/IntersectMBO/cardano-node/releases/download/8.9.1/cardano-node-8.9.1-linux.tar.gz
+ARG CARDANO_NODE_BUILD_URL_ARM64=hxxp://temporarily-broken
+
 RUN apt-get update -y && \
-  apt-get install -y wget tar && \
-  wget $CARDANO_NODE_BUILD_URL -O cardano-node.tar.gz && \
-  mkdir -p bin && \
-  tar -xzf cardano-node.tar.gz
+    apt-get install -y wget tar curl && \
+    if [ "$(uname -m)" = "aarch64" ] ; then \
+      curl -fsSL "$CARDANO_NODE_BUILD_URL_ARM64" >cardano-node.tar.gz ;\
+    else \
+      curl -fsSL "$CARDANO_NODE_BUILD_URL" >cardano-node.tar.gz ;\
+    fi && \
+    mkdir -p cardano-node && \
+    tar -xzf cardano-node.tar.gz -C cardano-node
 
 FROM ubuntu:${UBUNTU_VERSION}
 
-ENV TINI_VERSION v0.19.0
-
 WORKDIR /root
 RUN apt-get update -y && \
-  apt-get install -y tzdata ca-certificates jq coreutils curl
+  apt-get install -y tzdata ca-certificates jq coreutils curl wget
 
 HEALTHCHECK --interval=5s --timeout=1s --retries=2000 --start-period=100ms \
   CMD /root/scripts/get-epoch.sh | awk '{ if ($0 >= "3") exit 0; else exit 1}'
 
 STOPSIGNAL SIGINT
-COPY --from=builder /build/bin ./bin
-ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static /tini
-RUN chmod +x /tini
-
-COPY . .
+COPY --from=builder /build/cardano-node /opt/cardano-node
+ARG TINI_VERSION=v0.19.0
+RUN mkdir -p ./bin && ln -s /opt/cardano-node/bin/* ./bin/ &&\
+    if [ "$(uname -m)" = "aarch64" ] ; then \
+      TINI_VARIANT=static-arm64 ;\
+    else \
+      TINI_VARIANT=static-amd64 ;\
+    fi &&\
+    curl -fsSL >/tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TINI_VARIANT} &&\
+    chmod +x /tini
+
+COPY scripts scripts
+COPY templates templates
 ENTRYPOINT ["/tini", "-g", "--", "/root/scripts/start.sh" ]

packages/e2e/local-network/scripts/make-babbage.sh

@@ -14,11 +14,29 @@ export PATH=$PWD/bin:$PATH
 
 source ./scripts/nodes-configuration.sh
 
-UNAME=$(uname -s) SED=
-case $UNAME in
-Darwin) SED="gsed" ;;
-Linux) SED="sed" ;;
-esac
+# We need this when running in Docker Desktop on macOS. `sed -i` doesn’t work well with VOLUMEs
+# there, unless it can create its temporary files outside of a VOLUME, which requires $TMPDIR.
+export TMPDIR="${TMPDIR:-/tmp}"
+export TMP="${TMP:-/tmp}"
+
+UNAME=$(uname -s)
+
+# Normal `sed -i` is a bit stubborn, and really wants to create its temporary files in the
+# directory of the target file. It is not a true in-place edit, and often braks permissions.
+# Let’s use this wrapper instead.
+sed_i() {
+    local tmpfile=$(mktemp)
+    local sed_bin=sed
+    if [ "$UNAME" == "Darwin" ] ; then sed_bin=gsed ; fi
+    if $sed_bin "$@" >"$tmpfile"; then
+        cat "$tmpfile" >"${@: -1}" # Replace the last argument file (in-place file) with tmpfile content
+        rm "$tmpfile"
+    else
+        echo "sed failed." >&2
+        rm "$tmpfile"
+        return 1
+    fi
+}
 
 case $(uname) in
 Darwin) date='gdate' ;;
@@ -109,7 +127,7 @@ cp templates/babbage/alonzo-babbage-test-genesis.json "${ROOT}/genesis.alonzo.sp
 cp templates/babbage/conway-babbage-test-genesis.json "${ROOT}/genesis.conway.spec.json"
 cp templates/babbage/byron-configuration.yaml "${ROOT}/configuration.yaml"
 
-$SED -i "${ROOT}/configuration.yaml" \
+sed_i \
   -e 's/Protocol: RealPBFT/Protocol: Cardano/' \
   -e 's|GenesisFile: genesis.json|ByronGenesisFile: genesis/byron/genesis.json|' \
   -e '/ByronGenesisFile/ aShelleyGenesisFile: genesis/shelley/genesis.json' \
@@ -119,7 +137,8 @@ $SED -i "${ROOT}/configuration.yaml" \
   -e 's/LastKnownBlockVersion-Major: 0/LastKnownBlockVersion-Major: 6/' \
   -e 's/LastKnownBlockVersion-Minor: 2/LastKnownBlockVersion-Minor: 0/' \
   -e "s/minSeverity: Info/minSeverity: ${CARDANO_NODE_LOG_LEVEL}/" \
-  -e "s/cardano.node.ChainDB: Notice/cardano.node.ChainDB: ${CARDANO_NODE_CHAINDB_LOG_LEVEL}/"
+  -e "s/cardano.node.ChainDB: Notice/cardano.node.ChainDB: ${CARDANO_NODE_CHAINDB_LOG_LEVEL}/" \
+  "${ROOT}/configuration.yaml"
 
 echo "" >>"${ROOT}/configuration.yaml"
 echo "PBftSignatureThreshold: 0.6" >>"${ROOT}/configuration.yaml"
@@ -248,8 +267,8 @@ for NODE in ${SP_NODES}; do
 done
 
 echo "Update start time in genesis files"
-$SED -i -E "s/\"startTime\": [0-9]+/\"startTime\": ${timeUnix}/" ${ROOT}/genesis/byron/genesis.json
-$SED -i -E "s/\"systemStart\": \".*\"/\"systemStart\": \"${timeISO}\"/" ${ROOT}/genesis/shelley/genesis.json
+sed_i -E "s/\"startTime\": [0-9]+/\"startTime\": ${timeUnix}/" ${ROOT}/genesis/byron/genesis.json
+sed_i -E "s/\"systemStart\": \".*\"/\"systemStart\": \"${timeISO}\"/" ${ROOT}/genesis/shelley/genesis.json
 
 byronGenesisHash=$(cardano-cli byron genesis print-genesis-hash --genesis-json ${ROOT}/genesis/byron/genesis.json)
 shelleyGenesisHash=$(cardano-cli genesis hash --genesis ${ROOT}/genesis/shelley/genesis.json)
@@ -261,10 +280,10 @@ echo "Shelley genesis hash: $shelleyGenesisHash"
 echo "Alonzo genesis hash: $alonzoGenesisHash"
 echo "Conway genesis hash: $conwayGenesisHash"
 
-$SED -i -E "s/ByronGenesisHash: '.*'/ByronGenesisHash: '${byronGenesisHash}'/" ${ROOT}/configuration.yaml
-$SED -i -E "s/ShelleyGenesisHash: '.*'/ShelleyGenesisHash: '${shelleyGenesisHash}'/" ${ROOT}/configuration.yaml
-$SED -i -E "s/AlonzoGenesisHash: '.*'/AlonzoGenesisHash: '${alonzoGenesisHash}'/" ${ROOT}/configuration.yaml
-$SED -i -E "s/ConwayGenesisHash: '.*'/ConwayGenesisHash: '${conwayGenesisHash}'/" ${ROOT}/configuration.yaml
+sed_i -E "s/ByronGenesisHash: '.*'/ByronGenesisHash: '${byronGenesisHash}'/" ${ROOT}/configuration.yaml
+sed_i -E "s/ShelleyGenesisHash: '.*'/ShelleyGenesisHash: '${shelleyGenesisHash}'/" ${ROOT}/configuration.yaml
+sed_i -E "s/AlonzoGenesisHash: '.*'/AlonzoGenesisHash: '${alonzoGenesisHash}'/" ${ROOT}/configuration.yaml
+sed_i -E "s/ConwayGenesisHash: '.*'/ConwayGenesisHash: '${conwayGenesisHash}'/" ${ROOT}/configuration.yaml
 
 # Create config folder
 rm -rf ./config/*
@@ -277,15 +296,17 @@ cp ./templates/babbage/db-sync-config.json ./config/network/cardano-db-sync/conf
 cp ./templates/babbage/node-config.json ./config/network/cardano-node/config.json
 cp ./templates/babbage/submit-api-config.json ./config/network/cardano-submit-api/config.json
 
-$SED -i -E "s/\"ByronGenesisHash\": \".*\"/\"ByronGenesisHash\": \"${byronGenesisHash}\"/" ./config/network/cardano-node/config.json
-$SED -i -E "s/\"ShelleyGenesisHash\": \".*\"/\"ShelleyGenesisHash\": \"${shelleyGenesisHash}\"/" ./config/network/cardano-node/config.json
-$SED -i -E "s/\"AlonzoGenesisHash\": \".*\"/\"AlonzoGenesisHash\": \"${alonzoGenesisHash}\"/" ./config/network/cardano-node/config.json
-$SED -i -E "s/\"ConwayGenesisHash\": \".*\"/\"ConwayGenesisHash\": \"${conwayGenesisHash}\"/" ./config/network/cardano-node/config.json
+sed_i -E "s/\"ByronGenesisHash\": \".*\"/\"ByronGenesisHash\": \"${byronGenesisHash}\"/" ./config/network/cardano-node/config.json
+sed_i -E "s/\"ShelleyGenesisHash\": \".*\"/\"ShelleyGenesisHash\": \"${shelleyGenesisHash}\"/" ./config/network/cardano-node/config.json
+sed_i -E "s/\"AlonzoGenesisHash\": \".*\"/\"AlonzoGenesisHash\": \"${alonzoGenesisHash}\"/" ./config/network/cardano-node/config.json
+sed_i -E "s/\"ConwayGenesisHash\": \".*\"/\"ConwayGenesisHash\": \"${conwayGenesisHash}\"/" ./config/network/cardano-node/config.json
 
 cp ./templates/babbage/topology.json ./config/network/cardano-node/topology.json
 # docker hostname in topology.json isn't working, so need to specify ip of local network
 CONTAINER_IP=$(hostname -I | xargs)
-$SED -i "s/172.17.0.1/$CONTAINER_IP/g" ./config/network/cardano-node/topology.json
+sed_i "s/172.17.0.1/$CONTAINER_IP/g" ./config/network/cardano-node/topology.json
+# Note: for some reason, the first cardano-node (on port 3001) isn’t immediately responsive to the outside world, so:
+sed_i "s/3001/3002/g" ./config/network/cardano-node/topology.json
 
 cp "${ROOT}"/genesis/byron/genesis.json ./config/network/cardano-node/genesis/byron.json
 cp "${ROOT}"/genesis/byron/genesis.json ./config/network/genesis/byron.json

packages/e2e/local-network/scripts/ogmios-start.sh

@@ -10,4 +10,4 @@
 while [ -f /sdk-ipc/prevent_ogmios ]; do sleep 10; done
 
 # Start the ogmios as normal
-/bin/ogmios "$@"
+exec /bin/ogmios "$@"

packages/e2e/package.json

@@ -44,7 +44,7 @@
     "test:web-extension:watch:run": "yarn test:web-extension:run --watch",
     "test:web-extension:watch": "run-s test:web-extension:build test:web-extension:watch:bg",
     "test:web-extension:watch:bg": "run-p test:web-extension:watch:build test:web-extension:watch:run",
-    "local-network:common": "DISABLE_DB_CACHE=${DISABLE_DB_CACHE:-true} SUBMIT_API_ARGS='--testnet-magic 888' USE_BLOCKFROST=false docker compose -p local-network-e2e -f docker-compose.yml -f ../../compose/common.yml $FILES up",
+    "local-network:common": "DISABLE_DB_CACHE=${DISABLE_DB_CACHE:-true} SUBMIT_API_ARGS='--testnet-magic 888' USE_BLOCKFROST=false __FIX_UMASK__=$(chmod -R a+r ../../compose/placeholder-secrets) docker compose -p local-network-e2e -f docker-compose.yml -f ../../compose/common.yml -f ../../compose/$(uname -m).yml $FILES up",
     "local-network:up": "FILES='' yarn local-network:common",
     "local-network:dev": "FILES='-f ../../compose/dev.yml' yarn local-network:common",
     "local-network:profile:up": "FILES='-f ../../compose/pg-agent.yml' yarn local-network:common",