Revise logging #707
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # The reason this is so complicated is because github actions cache for docker | |
| # build doesn't seem to work as well as it should. Ideally it just caches the | |
| # work it has done before but I couldn't get this to work with much trying. | |
| # So we manually compute the hash of the files which the base docker image layer | |
| # depends upon and use this to tag images. If for some reason at some point | |
| # the github actions docker cache works more efficiently, we can simplyify. | |
| # The goals are: | |
| # - reproducible builds | |
| # - fast tests | |
| # - fast deployments | |
| name: node | |
| on: | |
| push: | |
| branches: | |
| - "master" | |
| pull_request: | |
| branches: | |
| - "master" | |
| permissions: | |
| contents: read # To check out code | |
| packages: write # To push to GitHub Container Registry | |
| jobs: | |
| setup-tests: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| image_hash: ${{ steps.hash.outputs.image_hash }} | |
| image_tag: ${{ steps.set-image-tag.outputs.image_tag }} | |
| env: | |
| # Define test suites here - single source of truth | |
| TEST_SUITES: | | |
| - app/blog | |
| - app/build | |
| - app/clients | |
| - app/dashboard | |
| - app/documentation | |
| - app/helper | |
| - app/models | |
| - app/site | |
| - app/sync | |
| - app/templates | |
| steps: | |
| - name: Sparse checkout for hash calculation | |
| uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: | | |
| Dockerfile | |
| package.json | |
| .dockerignore | |
| .github/workflows/node.yml | |
| sparse-checkout-cone-mode: false | |
| - name: Calculate hash of build-related files | |
| id: hash | |
| run: | | |
| # Create a list of files to hash | |
| FILES_TO_HASH="Dockerfile package.json .dockerignore .github/workflows/node.yml" | |
| echo "Hashing files: $FILES_TO_HASH" | |
| # Calculate hash of all these files combined | |
| HASH=$(cat $FILES_TO_HASH 2>/dev/null | sha256sum | cut -d ' ' -f 1) | |
| echo "image_hash=$HASH" >> $GITHUB_OUTPUT | |
| echo "Hash: $HASH" | |
| - name: Set image tag | |
| id: set-image-tag | |
| run: | | |
| IMAGE_TAG="ghcr.io/${{ github.repository_owner }}/blot:test-${{ steps.hash.outputs.image_hash }}" | |
| echo "image_tag=$IMAGE_TAG" >> $GITHUB_OUTPUT | |
| echo "Image tag: $IMAGE_TAG" | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Check if image exists | |
| id: check-image | |
| run: | | |
| IMAGE_TAG="${{ steps.set-image-tag.outputs.image_tag }}" | |
| if docker manifest inspect "$IMAGE_TAG" > /dev/null 2>&1; then | |
| echo "Image $IMAGE_TAG exists, skipping build" | |
| echo "image_exists=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Image $IMAGE_TAG does not exist, will build it" | |
| echo "image_exists=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Checkout full code for build | |
| if: steps.check-image.outputs.image_exists != 'true' | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| if: steps.check-image.outputs.image_exists != 'true' | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build test image | |
| if: steps.check-image.outputs.image_exists != 'true' | |
| uses: docker/build-push-action@v6 | |
| with: | |
| platforms: linux/amd64 | |
| target: dev | |
| context: . | |
| push: true | |
| cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/blot | |
| cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/blot,mode=max | |
| tags: | | |
| ${{ steps.set-image-tag.outputs.image_tag }} | |
| ghcr.io/${{ github.repository_owner }}/blot:test-latest | |
| - name: Convert test suites to matrix | |
| id: set-matrix | |
| run: | | |
| # Read test suites from environment variable | |
| echo "Test suites defined:" | |
| echo "$TEST_SUITES" | |
| # Clean the input - remove any leading/trailing whitespace from each line | |
| CLEANED_TEST_SUITES=$(echo "$TEST_SUITES" | sed -e 's/^[ \t]*//' -e 's/[ \t]*$//') | |
| # Process the YAML list into a proper JSON array | |
| TEST_SUITES_JSON=$(echo "$CLEANED_TEST_SUITES" | grep '^- ' | sed 's/^- /"/g' | sed 's/$/"/g' | tr '\n' ',' | sed 's/,$//' | sed 's/^/[/' | sed 's/$/]/') | |
| # Format as JSON for GitHub Actions matrix - ensure it's valid JSON | |
| echo "Creating matrix JSON..." | |
| MATRIX_JSON="{\"test_suite\":$TEST_SUITES_JSON}" | |
| echo "$MATRIX_JSON" | jq '.' # Validate JSON format | |
| echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT | |
| detect-unexecuted-specs: | |
| needs: setup-tests | |
| runs-on: ubuntu-latest | |
| env: | |
| # Copy the SKIP_PATHS definition from setup-tests | |
| SKIP_PATHS: | | |
| - config/openresty | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Extract test suites from matrix | |
| run: | | |
| # Extract test suites directly from the matrix JSON | |
| echo "Test suites defined in matrix:" | |
| MATRIX_JSON='${{ needs.setup-tests.outputs.matrix }}' | |
| echo "$MATRIX_JSON" | |
| # Extract the test_suite array from the matrix JSON | |
| TEST_SUITES=$(echo "$MATRIX_JSON" | jq -r '.test_suite[]') | |
| echo "Extracted test suites:" | |
| echo "$TEST_SUITES" | |
| # Save to file for validation script | |
| echo "$TEST_SUITES" > matrix_suites.txt | |
| - name: Find all spec files | |
| run: | | |
| # First find files containing "describe(" | |
| echo "Finding all potential spec files..." | |
| DESCRIBE_FILES=$(grep -r --include="*.js" -l "describe(" . | | |
| grep -v "node_modules" || true) | |
| echo "Files with describe():" | |
| echo "$DESCRIBE_FILES" | |
| # Then among those, find files also containing "it(" | |
| if [ -n "$DESCRIBE_FILES" ]; then | |
| echo "Filtering for files with both describe() and it()..." | |
| SPEC_FILES=$(echo "$DESCRIBE_FILES" | xargs grep -l "it(" || true) | |
| else | |
| SPEC_FILES="" | |
| fi | |
| echo "Files with both describe() and it() - pre-filtering:" | |
| echo "$SPEC_FILES" | |
| # Process and apply skip paths | |
| echo "Processing skip paths..." | |
| FINAL_SPEC_FILES="" | |
| while IFS= read -r file; do | |
| if [ -z "$file" ]; then continue; fi | |
| SKIP=false | |
| # Check each file against all skip paths | |
| while IFS= read -r skip_path; do | |
| skip_path=$(echo "$skip_path" | sed 's/^- //') | |
| if [ -z "$skip_path" ]; then continue; fi | |
| if [[ "$file" == *"$skip_path"* ]]; then | |
| echo "Skipping file: $file (matches skip path: $skip_path)" | |
| SKIP=true | |
| break | |
| fi | |
| done < <(echo "$SKIP_PATHS" | grep "^- ") | |
| # Add file to final list if not skipped | |
| if [[ "$SKIP" != "true" ]]; then | |
| FINAL_SPEC_FILES+="$file"$'\n' | |
| fi | |
| done < <(echo "$SPEC_FILES") | |
| echo "Final spec files (after applying filters):" | |
| echo "$FINAL_SPEC_FILES" | |
| # Save the list to a file for further processing | |
| echo "$FINAL_SPEC_FILES" > all_spec_files.txt | |
| - name: Check if all specs are executed by test matrix | |
| run: | | |
| if [ ! -s all_spec_files.txt ]; then | |
| echo "No spec files found or all have been filtered out. Nothing to check." | |
| exit 0 | |
| fi | |
| # Create a regex pattern from the matrix suites for RUNNABLE test files | |
| # This matches files in /tests/ directories or named tests.js within the matrix suites | |
| PATTERN="" | |
| while read -r suite; do | |
| if [ -n "$suite" ]; then | |
| # Only match files in /tests/ directories or named tests.js within each suite | |
| PATTERN="${PATTERN}|^\./${suite}/.*/tests/.*\.js|^\./${suite}/.*tests\.js|^\./${suite}/tests/.*\.js" | |
| fi | |
| done < matrix_suites.txt | |
| if [ -z "$PATTERN" ]; then | |
| echo "No test suites defined in matrix. Cannot perform check." | |
| exit 1 | |
| fi | |
| PATTERN="${PATTERN:1}" # Remove the leading | | |
| echo "Test execution pattern: $PATTERN" | |
| # Find spec files not matching the execution pattern | |
| echo "Checking for spec files not executed by any test suite in the matrix..." | |
| UNEXECUTED_SPECS=$(grep -v -E "$PATTERN" all_spec_files.txt || true) | |
| if [ -n "$UNEXECUTED_SPECS" ]; then | |
| echo "WARNING: Found spec files that contain tests but won't be executed by the test runner:" | |
| echo "$UNEXECUTED_SPECS" | |
| echo "" | |
| echo "These files contain describe() and it() blocks but aren't in a /tests/ directory or named tests.js," | |
| echo "so they won't be picked up by the test runner within their respective test suites." | |
| echo "" | |
| echo "Please either:" | |
| echo "1. Move them to a /tests/ directory" | |
| echo "2. Rename them to tests.js" | |
| echo "3. Add their directories to SKIP_PATHS if they should be skipped" | |
| exit 1 | |
| else | |
| echo "All spec files are properly located to be executed by the test runner. Good job!" | |
| fi | |
| test: | |
| needs: setup-tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: ${{ fromJson(needs.setup-tests.outputs.matrix) }} | |
| fail-fast: false | |
| services: | |
| redis: | |
| image: redis:6 | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run tests - ${{ matrix.test_suite }} | |
| env: | |
| BLOT_STRIPE_KEY: ${{ secrets.BLOT_STRIPE_KEY }} | |
| BLOT_STRIPE_SECRET: ${{ secrets.BLOT_STRIPE_SECRET }} | |
| BLOT_STRIPE_PRODUCT: ${{ secrets.BLOT_STRIPE_PRODUCT }} | |
| IMAGE_TAG: ${{ needs.setup-tests.outputs.image_tag }} | |
| run: | | |
| docker run --rm \ | |
| --network host \ | |
| -e BLOT_REDIS_HOST=localhost \ | |
| -e BLOT_STRIPE_KEY=$BLOT_STRIPE_KEY \ | |
| -e BLOT_STRIPE_SECRET=$BLOT_STRIPE_SECRET \ | |
| -e BLOT_STRIPE_PRODUCT=$BLOT_STRIPE_PRODUCT \ | |
| -v ${{ github.workspace }}/app:/usr/src/app/app \ | |
| -v ${{ github.workspace }}/scripts:/usr/src/app/scripts \ | |
| -v ${{ github.workspace }}/config:/usr/src/app/config \ | |
| -v ${{ github.workspace }}/notes:/usr/src/app/notes \ | |
| -v ${{ github.workspace }}/todo.txt:/usr/src/app/todo.txt \ | |
| -v ${{ github.workspace }}/.git:/usr/src/app/.git \ | |
| -v ${{ github.workspace }}/tests:/usr/src/app/tests \ | |
| $IMAGE_TAG \ | |
| sh -c "node /usr/src/app/app/documentation/build/index.js --no-watch --skip-zip && node tests ${{ matrix.test_suite }} && npx depcheck --ignores=depcheck,nyc,nodemon,blessed-contrib,fontkit,text-to-svg --skip-missing" | |
| build-production-images: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build production image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| platforms: linux/amd64,linux/arm64 | |
| target: prod | |
| context: . | |
| push: true | |
| cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/blot | |
| cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/blot,mode=max | |
| tags: ghcr.io/${{ github.repository_owner }}/blot:${{ github.sha }} | |
| - name: Verify built-in health check | |
| env: | |
| IMAGE: ghcr.io/${{ github.repository_owner }}/blot:${{ github.sha }} | |
| run: | | |
| docker network create test_network | |
| # Start a Redis container | |
| redis_container_id=$(docker run -d --name test_redis --network test_network redis:latest) | |
| # Ensure Redis started successfully | |
| if [ -z "$redis_container_id" ]; then | |
| echo "Failed to start the Redis container. Exiting..." | |
| exit 1 | |
| fi | |
| echo "Waiting for Redis ($redis_container_id) to become ready..." | |
| timeout=30 | |
| interval=2 | |
| elapsed=0 | |
| while ! docker exec $redis_container_id redis-cli ping | grep -q PONG; do | |
| if [ $elapsed -ge $timeout ]; then | |
| echo "Redis did not become ready within $timeout seconds. Exiting..." | |
| docker stop $redis_container_id | |
| docker rm $redis_container_id | |
| exit 1 | |
| fi | |
| sleep $interval | |
| elapsed=$((elapsed + interval)) | |
| done | |
| echo "Redis is ready." | |
| # Start the app container with Redis environment variables | |
| container_id=$(docker run -d --network test_network --env BLOT_REDIS_HOST=test_redis -p 8080:8080 $IMAGE) | |
| # Ensure the app container started successfully | |
| if [ -z "$container_id" ]; then | |
| echo "Failed to start the app container. Exiting..." | |
| docker stop $redis_container_id | |
| docker rm $redis_container_id | |
| exit 1 | |
| fi | |
| echo "Waiting for the app container ($container_id) to pass the built-in health check..." | |
| # Wait for the app container's health status to become "healthy" | |
| timeout=60 | |
| interval=5 | |
| elapsed=0 | |
| while [ "$(docker inspect --format='{{json .State.Health.Status}}' $container_id)" != '"healthy"' ]; do | |
| if [ $elapsed -ge $timeout ]; then | |
| echo "Health check failed: app container did not become healthy within $timeout seconds." | |
| echo "Final log contents:" | |
| docker exec $container_id cat /usr/src/app/data/logs/docker/app.log | |
| docker stop $container_id | |
| docker rm $container_id | |
| docker stop $redis_container_id | |
| docker rm $redis_container_id | |
| exit 1 | |
| fi | |
| sleep $interval | |
| elapsed=$((elapsed + interval)) | |
| done | |
| echo "App container passed the health check." | |
| # Clean up | |
| docker stop $container_id | |
| docker rm $container_id | |
| docker stop $redis_container_id | |
| docker rm $redis_container_id |