Switches unittest to pytest for testing (#2459)

# Description

Switches our unit testing framework from unittest to pytest to enable
better CI logging and improve developer experience when checking for
test failures.

## Type of change

- Bug fix (non-breaking change which fixes an issue)
- New feature (non-breaking change which adds functionality)
- Breaking change (fix or feature that would cause existing
functionality to not work as expected)

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [ ] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

<!--
As you go through the checklist above, you can mark something as done by
putting an x character in it

For example,
- [x] I have done this task
- [ ] I have not done this task
-->

---------

Co-authored-by: Pascal Roth <[email protected]>
Co-authored-by: Pascal Roth <[email protected]>

Author: kellyguo11

.aws/premerge-ci-buildspec.yml

@@ -1,83 +1,305 @@
+# Copyright (c) 2022-2025, The Isaac Lab Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# This buildspec file defines the CI/CD pipeline for IsaacLab.
+# It runs tests on an EC2 instance with GPU support and uses Docker BuildKit
+# for efficient builds with S3 caching.
+#
+# Required environment variables:
+# - ISAACSIM_BASE_IMAGE: Base image for IsaacSim
+# - ISAACSIM_BASE_VERSION: Version of IsaacSim to use
+#
+# Required AWS Secrets:
+# - NGC_TOKEN: NVIDIA NGC authentication token
+# - SSH_KEY: SSH private key for EC2 access
+# - SSH_PUBLIC_KEY: SSH public key for EC2 access
+
 version: 0.2
 
+env:
+  variables:
+    # Build configuration
+    MAX_RETRIES: "5"
+    RETRY_WAIT_TIME: "30"
+
+    # EC2 configuration
+    INSTANCE_TYPE: "g5.2xlarge"
+    VOLUME_SIZE: "500"
+    REGION: "us-west-2"
+    AZ: "us-west-2a"
+
+    # Docker and cache configuration
+    ECR_REPOSITORY: "isaaclab-dev"
+    CACHE_BUCKET_PREFIX: "isaaclab-build-cache"
+
+    # Docker versions
+    BUILDX_VERSION: "0.11.2"
+
+  secrets-manager:
+    NGC_TOKEN: "production/ngc/token"
+    SSH_KEY: "production/ssh/isaaclab"
+    SSH_PUBLIC_KEY: "production/ssh/isaaclab"
+
 phases:
+  install:
+    runtime-versions:
+      python: 3.9
+    commands:
+      - echo "Installing required packages..."
+      - pip install awscli boto3
+
   pre_build:
     commands:
-      - echo "Launching EC2 instance to run tests"
       - |
+        # Validate required environment variables
+        if [ -z "$ISAACSIM_BASE_IMAGE" ]; then
+          echo "Error: Required environment variable ISAACSIM_BASE_IMAGE is not set"
+          exit 1
+        fi
+        if [ -z "$ISAACSIM_BASE_VERSION" ]; then
+          echo "Error: Required environment variable ISAACSIM_BASE_VERSION is not set"
+          exit 1
+        fi
+
+        # Get AWS account ID
+        AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+        if [ -z "$AWS_ACCOUNT_ID" ]; then
+          echo "Error: Failed to get AWS account ID"
+          exit 1
+        fi
+
+        # Create ECR repository if it doesn't exist
+        aws ecr describe-repositories --repository-names $ECR_REPOSITORY || \
+          aws ecr create-repository --repository-name $ECR_REPOSITORY
+
+        # Configure ECR repository lifecycle policy
+        aws ecr put-lifecycle-policy \
+          --repository-name $ECR_REPOSITORY \
+          --lifecycle-policy-text '{
+            "rules": [
+              {
+                "rulePriority": 1,
+                "description": "Expire images older than 2 weeks",
+                "selection": {
+                  "tagStatus": "any",
+                  "countType": "sinceImagePushed",
+                  "countUnit": "days",
+                  "countNumber": 14
+                },
+                "action": {
+                  "type": "expire"
+                }
+              }
+            ]
+          }'
+
+        # Create S3 bucket for BuildKit cache if it doesn't exist
+        CACHE_BUCKET="${CACHE_BUCKET_PREFIX}-${AWS_ACCOUNT_ID}"
+        aws s3api head-bucket --bucket $CACHE_BUCKET || \
+          aws s3 mb s3://$CACHE_BUCKET --region $REGION
+
+        # Configure S3 bucket lifecycle rule for cache expiration
+        aws s3api put-bucket-lifecycle-configuration \
+          --bucket $CACHE_BUCKET \
+          --lifecycle-configuration '{
+            "Rules": [
+              {
+                "ID": "ExpireBuildKitCache",
+                "Status": "Enabled",
+                "Filter": {
+                  "Prefix": ""
+                },
+                "Expiration": {
+                  "Days": 14
+                }
+              }
+            ]
+          }'
+
+        echo "Launching EC2 instance to run tests..."
         INSTANCE_ID=$(aws ec2 run-instances \
           --image-id ami-0e6cc441f9f4caab3 \
           --count 1 \
-          --instance-type g5.2xlarge \
+          --instance-type $INSTANCE_TYPE \
           --key-name production/ssh/isaaclab \
           --security-group-ids sg-02617e4b8916794c4 \
           --subnet-id subnet-0907ceaeb40fd9eac \
-          --block-device-mappings '[{"DeviceName":"/dev/sda1","Ebs":{"VolumeSize":500}}]' \
+          --iam-instance-profile Name="IsaacLabBuildRole" \
+          --block-device-mappings "[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"VolumeSize\":$VOLUME_SIZE}}]" \
           --output text \
           --query 'Instances[0].InstanceId')
-      - aws ec2 wait instance-running --instance-ids $INSTANCE_ID
-      - |
+
+        echo "Waiting for instance $INSTANCE_ID to be running..."
+        aws ec2 wait instance-running --instance-ids $INSTANCE_ID
+
+        echo "Getting instance IP address..."
         EC2_INSTANCE_IP=$(aws ec2 describe-instances \
           --filters "Name=instance-state-name,Values=running" "Name=instance-id,Values=$INSTANCE_ID" \
           --query 'Reservations[*].Instances[*].[PrivateIpAddress]' \
           --output text)
-      - mkdir -p ~/.ssh
-      - |
+
+        echo "Setting up SSH configuration..."
+        mkdir -p ~/.ssh
         aws ec2 describe-key-pairs --include-public-key --key-name production/ssh/isaaclab \
           --query 'KeyPairs[0].PublicKey' --output text > ~/.ssh/id_rsa.pub
-      - |
-        aws secretsmanager get-secret-value --secret-id production/ssh/isaaclab \
-          --query SecretString --output text > ~/.ssh/id_rsa
-      - chmod 400 ~/.ssh/id_*
-      - echo "Host $EC2_INSTANCE_IP\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
-      - |
+        echo "$SSH_KEY" > ~/.ssh/id_rsa
+        chmod 400 ~/.ssh/id_*
+        echo "Host $EC2_INSTANCE_IP\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile=/dev/null\n" >> ~/.ssh/config
+
+        echo "Sending SSH public key to instance..."
         aws ec2-instance-connect send-ssh-public-key \
           --instance-id $INSTANCE_ID \
-          --availability-zone us-west-2a \
+          --availability-zone $AZ \
           --ssh-public-key file://~/.ssh/id_rsa.pub \
           --instance-os-user ubuntu
+
   build:
     commands:
-      - echo "Running tests on EC2 instance"
-      - SRC_DIR=$(basename $CODEBUILD_SRC_DIR)
-      - cd ..
       - |
-        bash -c '
-        function retry_scp() {
-          local retries=5
-          local wait_time=30
-          local count=0
-          while [ $count -lt $retries ]; do
+        #!/bin/sh
+        set -e
+
+        echo "Running tests on EC2 instance..."
+        SRC_DIR=$(basename $CODEBUILD_SRC_DIR)
+        cd ..
+
+        # Retry SCP with exponential backoff
+        retry_count=0
+        wait_time=$RETRY_WAIT_TIME
+
+        while [ $retry_count -lt $MAX_RETRIES ]; do
+          if [ $retry_count -gt 0 ]; then
+            wait_time=$((wait_time * 2))
+            echo "Retry attempt $((retry_count + 1))/$MAX_RETRIES. Waiting $wait_time seconds..."
             sleep $wait_time
-            scp -r $SRC_DIR ubuntu@$EC2_INSTANCE_IP:~
-            if [ $? -eq 0 ]; then
-              echo "SCP command succeeded"
-              return 0
-            fi
-            count=$((count + 1))
-            echo "SCP command failed. Retrying in $wait_time seconds..."
-          done
-          echo "SCP command failed after $retries attempts."
-          return 1
-        }
-        retry_scp
-        '
-      - ssh ubuntu@$EC2_INSTANCE_IP "docker login -u \\\$oauthtoken -p $NGC_TOKEN nvcr.io"
-      - |
-        ssh ubuntu@$EC2_INSTANCE_IP "
+          fi
+
+          if scp -o ConnectTimeout=10 -o StrictHostKeyChecking=no -r $SRC_DIR ubuntu@$EC2_INSTANCE_IP:~; then
+            echo "SCP command succeeded"
+            break
+          fi
+
+          retry_count=$((retry_count + 1))
+        done
+
+        if [ $retry_count -eq $MAX_RETRIES ]; then
+          echo "SCP command failed after $MAX_RETRIES attempts"
+          exit 1
+        fi
+
+        # Get ECR login token
+        ECR_LOGIN_TOKEN=$(aws ecr get-login-password --region $REGION)
+
+        # Run tests with proper error handling and Docker caching
+        ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no ubuntu@$EC2_INSTANCE_IP "
+          set -e
+
+          # Install Docker with BuildKit support
+          echo 'Installing Docker with BuildKit support...'
+          sudo apt-get update
+          sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+          sudo add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\"
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+
+          # Enable BuildKit at daemon level
+          sudo mkdir -p /etc/docker
+          echo '{\"features\":{\"buildkit\":true}}' | sudo tee /etc/docker/daemon.json
+
+          # Install Docker Buildx
+          echo 'Installing Docker Buildx...'
+          mkdir -p ~/.docker/cli-plugins/
+          curl -SL https://github.com/docker/buildx/releases/download/v$BUILDX_VERSION/buildx-v$BUILDX_VERSION.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx
+          chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+          # Add current user to docker group
+          sudo usermod -aG docker ubuntu
+          newgrp docker
+
+          echo 'Logging into NGC...'
+          docker login -u \\\$oauthtoken -p $NGC_TOKEN nvcr.io
+
+          # Login to ECR using token from CodeBuild
+          echo \"$ECR_LOGIN_TOKEN\" | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com
+
           cd $SRC_DIR
-          DOCKER_BUILDKIT=1 docker build -t isaac-lab-dev \
-            --build-arg ISAACSIM_BASE_IMAGE_ARG=$ISAACSIM_BASE_IMAGE \
-            --build-arg ISAACSIM_VERSION_ARG=$ISAACSIM_BASE_VERSION \
-            --build-arg ISAACSIM_ROOT_PATH_ARG=/isaac-sim \
-            --build-arg ISAACLAB_PATH_ARG=/workspace/isaaclab \
-            --build-arg DOCKER_USER_HOME_ARG=/root \
-            -f docker/Dockerfile.base .
+          echo 'Building Docker image with BuildKit caching...'
+
+          # Configure BuildKit environment
+          export DOCKER_BUILDKIT=1
+          export BUILDKIT_INLINE_CACHE=1
+
+          # Create a new builder instance with S3 cache support
+          docker buildx create --name mybuilder --driver docker-container --bootstrap
+          docker buildx use mybuilder
+
+          # Build with BuildKit and S3 cache
+          if docker pull $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$ECR_REPOSITORY:latest 2>/dev/null; then
+            echo "Using existing image for cache..."
+            docker buildx build --progress=plain --platform linux/amd64 -t isaac-lab-dev \
+              --cache-from type=registry,ref=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$ECR_REPOSITORY:latest \
+              --cache-to type=s3,region=$REGION,bucket=$CACHE_BUCKET,mode=max,ignore-error=true \
+              --build-arg ISAACSIM_BASE_IMAGE_ARG=$ISAACSIM_BASE_IMAGE \
+              --build-arg ISAACSIM_VERSION_ARG=$ISAACSIM_BASE_VERSION \
+              --build-arg ISAACSIM_ROOT_PATH_ARG=/isaac-sim \
+              --build-arg ISAACLAB_PATH_ARG=/workspace/isaaclab \
+              --build-arg DOCKER_USER_HOME_ARG=/root \
+              -f docker/Dockerfile.base \
+              --load .
+          else
+            echo "No existing image found, building without cache-from..."
+            docker buildx build --progress=plain --platform linux/amd64 -t isaac-lab-dev \
+              --cache-to type=s3,region=$REGION,bucket=$CACHE_BUCKET,mode=max,ignore-error=true \
+              --build-arg ISAACSIM_BASE_IMAGE_ARG=$ISAACSIM_BASE_IMAGE \
+              --build-arg ISAACSIM_VERSION_ARG=$ISAACSIM_BASE_VERSION \
+              --build-arg ISAACSIM_ROOT_PATH_ARG=/isaac-sim \
+              --build-arg ISAACLAB_PATH_ARG=/workspace/isaaclab \
+              --build-arg DOCKER_USER_HOME_ARG=/root \
+              -f docker/Dockerfile.base \
+              --load .
+          fi
+
+          echo 'Running tests...'
+          TEST_EXIT_CODE=0
           docker run --rm --entrypoint bash --gpus all --network=host \
-            --name isaac-lab-test isaac-lab-dev ./isaaclab.sh -t && exit \$?
-        "
+            --name isaac-lab-test -v ~/$SRC_DIR/reports:/workspace/IsaacLab/tests isaac-lab-dev \
+            /isaac-sim/python.sh -m \
+            pytest tools -v || TEST_EXIT_CODE=$?
+
+          echo "Test exit code: $TEST_EXIT_CODE" > ~/$SRC_DIR/test_exit_code.txt
+        " || { echo "Test execution failed"; exit 1; }
+
+        echo "Copying test reports..."
+        mkdir -p $CODEBUILD_SRC_DIR/reports
+        scp -o ConnectTimeout=10 -o StrictHostKeyChecking=no -r ubuntu@$EC2_INSTANCE_IP:~/$SRC_DIR/reports/test-reports.xml $CODEBUILD_SRC_DIR/reports/
+        scp -o ConnectTimeout=10 -o StrictHostKeyChecking=no ubuntu@$EC2_INSTANCE_IP:~/$SRC_DIR/test_exit_code.txt $CODEBUILD_SRC_DIR/
+
+        if [ "$(cat $CODEBUILD_SRC_DIR/test_exit_code.txt)" != "0" ]; then
+          echo "Tests failed with exit code $(cat $CODEBUILD_SRC_DIR/test_exit_code.txt)"
+          exit 1
+        fi
 
   post_build:
     commands:
-      - echo "Terminating EC2 instance"
-      - aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+      - |
+        echo "Cleaning up resources..."
+        if [ ! -z "$INSTANCE_ID" ]; then
+          echo "Terminating EC2 instance $INSTANCE_ID..."
+          aws ec2 terminate-instances --instance-ids $INSTANCE_ID || true
+        fi
+
+reports:
+  pytest_reports:
+    files:
+      - 'reports/test-reports.xml'
+    base-directory: '.'
+    file-format: JUNITXML
+
+cache:
+  paths:
+    - '/root/.cache/pip/**/*'
+    - '/root/.docker/**/*'
+    - '/root/.aws/**/*'

.gitignore

@@ -63,3 +63,6 @@ _build
 
 # Teleop Recorded Dataset
 datasets
+
+# Tests
+tests/

.vscode/tools/launch.template.json

@@ -33,6 +33,25 @@
             "args" : ["--task", "Isaac-Reach-Franka-v0", "--num_envs", "32"],
             "program": "${workspaceFolder}/scripts/reinforcement_learning/rsl_rl/play.py",
             "console": "integratedTerminal"
+        },
+        {
+            "name": "Python: SinglePytest",
+            "type": "python",
+            "request": "launch",
+            "module": "pytest",
+            "args": [
+              "${file}"
+            ],
+            "console": "integratedTerminal"
+        },
+        {
+            "name": "Python: ALL Pytest",
+            "type": "python",
+            "request": "launch",
+            "module": "pytest",
+            "args": ["source/isaaclab/test"],
+            "console": "integratedTerminal",
+            "justMyCode": false
         }
     ]
 }