From e523092946f907ff61fcb44159e8a56137f59574 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 27 Apr 2023 16:48:03 +0200 Subject: [PATCH 1/9] Add a testing CI on the aws image --- .github/scripts/check_instance.sh | 17 +++++ .../scripts/check_meilisearch_availability.sh | 16 +++++ .github/workflows/test.yml | 62 +++++++++++++++++++ .gitignore | 15 ++++- .terraform.lock.hcl | 24 +++++++ terraform-config.tf | 39 ++++++++++++ 6 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/check_instance.sh create mode 100644 .github/scripts/check_meilisearch_availability.sh create mode 100644 .github/workflows/test.yml create mode 100644 .terraform.lock.hcl create mode 100644 terraform-config.tf diff --git a/.github/scripts/check_instance.sh b/.github/scripts/check_instance.sh new file mode 100644 index 0000000..cde2f3e --- /dev/null +++ b/.github/scripts/check_instance.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +start_time=$(date +%s) + +while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' --max-time 5 http://11.1.1)" != "200" ]]; do + sleep 5 + + elapsed_time=$(($(date +%s) - $start_time)) + # A timeout error is raised after waiting for 10 minutes + if [[ $elapsed_time -gt 600 ]]; then + echo "Timeout error: The request took too long to complete." + exit 1 + fi +done + +echo "Instance is ready!" +exit 0 diff --git a/.github/scripts/check_meilisearch_availability.sh b/.github/scripts/check_meilisearch_availability.sh new file mode 100644 index 0000000..02fbd30 --- /dev/null +++ b/.github/scripts/check_meilisearch_availability.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +PUBLIC_IP=$1 +status="$(curl http://$PUBLIC_IP/health)" + +# Check if the variable content is equal to '{"status":"available"}' +if [ "$status" != '{"status":"available"}' ]; then + # If not, echo the variable value and an error message, and exit with code 1 + echo "Error: Meilisearch is not running correctly." + echo "Server response: $status" + exit 1 +else + # If it is equal, echo a success message and exit with code 0 + echo "Meilisearch is available." + exit 0 +fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..062c0bd --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,62 @@ +name: Test AWS image + +on: + pull_request: + push: + # trying and staging branches are for BORS config + branches: + - trying + - staging + - main + +jobs: + test-image: + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Install Packer + run: | + wget https://releases.hashicorp.com/packer/1.7.7/packer_1.7.7_linux_amd64.zip # change version + unzip packer_1.7.7_linux_amd64.zip + sudo mv packer /usr/local/bin/ + + - name: Build image + run: | + image_id=$(packer build -var 'image_name=temporary-iam-used-in-packer-testing' -machine-readable -only "amazon-ebs.*" meilisearch.pkr.hcl | awk -F, '$0 ~/artifact,0,id/ {print $6}' | cut -d ":" -f 2) + echo $image_id + echo "IMAGE_ID=$image_id" >> $GITHUB_ENV + + - name: check env + run: echo ${{ env.IMAGE_ID }} + + - name: Install Terraform + run: | + sudo apt-get update + sudo apt-get install -y curl unzip + curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - + sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + sudo apt-get update && sudo apt-get install -y terraform + + - name: Create instance + run: | + terraform init + terraform apply -auto-approve -var ami_id=$IMAGE_ID + public_ip=$(terraform output public_ip) + echo "PUBLIC_IP=$public_ip" >> $GITHUB_ENV + - name: Test that the instance is running + run: | + sh .github/scripts/check-instance.sh ${{ env.PUBLIC_IP }} + - name: Test that Meilisearch is running correctly + run: | + sh .github/scripts/check_meilisearch_availability.sh ${{ env.PUBLIC_IP }} + - name: Destroy instance and AMI + if: always() + run: | + terraform destroy -var ami_id=$IMAGE_ID -auto-approve + + \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2cbc1ad..7e972f6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,12 +5,21 @@ packer_cache/ crash.log # https://www.packer.io/guides/hcl/variables -# Exclude all .pkrvars.hcl files, which are likely to contain sensitive data, -# such as password, private keys, and other secrets. These should not be part of -# version control as they are data points which are potentially sensitive and +# Exclude all .pkrvars.hcl files, which are likely to contain sensitive data, +# such as password, private keys, and other secrets. These should not be part of +# version control as they are data points which are potentially sensitive and # subject to change depending on the environment. # *.pkrvars.hcl # For built boxes *.box + +**/terraform.plan + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl new file mode 100644 index 0000000..a760b5d --- /dev/null +++ b/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.64.0" + hashes = [ + "h1:xstX5ub6MZ+PSrrZbB0ElhThX8N2ShQThR3m8nMZ928=", + "zh:092614f767995140cf444cad1a97fb569885db16cb1c1dc9ee56e801232bac29", + "zh:142e262fbb162c8a86493cfab4aadaf96a8572f1a3a6be444d465a4aee377dba", + "zh:1c58c8cb9934dc98a2dd9dc48a8a3d94a14c2c3f2bc0136410a9344938d4ecfb", + "zh:36efdf30cd52b92668cf6f912538c6e176b1a140a00e63ee0f753b85878c8b53", + "zh:4c631e367fd69692b57f85564de561733380e9674e146d3a7725b781ec5db944", + "zh:57ace91cb022ec944ad3af9272b78f48e7f71e9d1bf113ca56c6ce8deb4341fe", + "zh:7fc9581b530ebf28fda80c62c20c6fbbb936a878c24872349eb107b7f198e64c", + "zh:8280cd8f04c31af83f3e74f07704b258fbaa8bf1d70679d5ea2f0cbda2571de2", + "zh:8e6217a9443b651d4349d75bdc37af9298970d854bf515d8c305919b193e4a38", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c62bc4a9034a6caf15b8863da6f5a621b947d5fca161b4bd2f2e8e78eec8e3b", + "zh:9d0a45cd4a031d19ee14c0a15f25df6359dcd342ccf4e2ee4751b3ee496edb57", + "zh:ab47f4e300c46dc1757e2b8d8d749f34f044f219479106a00bf40572091a8999", + "zh:b55119290497dda96ab9ba3dca00d648808dc99d18960ad8aa875775bfaf95db", + "zh:df513941e6979f557edcac28d84bd91af9786104b0deba45b3b259a5ad215897", + ] +} diff --git a/terraform-config.tf b/terraform-config.tf new file mode 100644 index 0000000..b86f8b0 --- /dev/null +++ b/terraform-config.tf @@ -0,0 +1,39 @@ +variable "ami_id" { + type = string +} + +variable "region" { + description = "The AWS region to use for resources" + type = string + default = "us-east-1" +} + +provider "aws" { + region = var.region +} + +data "aws_security_group" "selected" { + id = "sg-037fd498b332442c1" +} + +resource "aws_instance" "test" { + ami = var.ami_id + instance_type = "t2.small" + tags = { + Name = "aws-packer-testing-instance" + } + vpc_security_group_ids = [data.aws_security_group.selected.id] + + provisioner "local-exec" { + command = "aws ec2 deregister-image --image-id ${self.ami}" + when = destroy + + environment = { + AWS_REGION = "us-east-1" + } + } +} + +output "public_ip_address" { + value = aws_instance.test.public_ip +} From dbdd90f5e1da4883430d761555736c63ae57c78e Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 27 Apr 2023 17:29:56 +0200 Subject: [PATCH 2/9] Fix test CI --- .github/scripts/check_instance.sh | 3 ++- .github/workflows/test.yml | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/scripts/check_instance.sh b/.github/scripts/check_instance.sh index cde2f3e..f829603 100644 --- a/.github/scripts/check_instance.sh +++ b/.github/scripts/check_instance.sh @@ -1,8 +1,9 @@ #!/bin/bash +PUBLIC_IP=$1 start_time=$(date +%s) -while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' --max-time 5 http://11.1.1)" != "200" ]]; do +while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' --max-time 5 http://$PUBLIC_IP)" != "200" ]]; do sleep 5 elapsed_time=$(($(date +%s) - $start_time)) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 062c0bd..cdc9455 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,10 @@ jobs: - name: Build image run: | - image_id=$(packer build -var 'image_name=temporary-iam-used-in-packer-testing' -machine-readable -only "amazon-ebs.*" meilisearch.pkr.hcl | awk -F, '$0 ~/artifact,0,id/ {print $6}' | cut -d ":" -f 2) + packer init . + image_id="$(packer build -var 'image_name=${{ github.run_id }}' -machine-readable -only "amazon-ebs.*" meilisearch.pkr.hcl | awk -F, '$0 ~/artifact,0,id/ {print $6}' | cut -d ":" -f 2)" + echo "hello" + echo "$image_id" echo $image_id echo "IMAGE_ID=$image_id" >> $GITHUB_ENV @@ -46,11 +49,11 @@ jobs: run: | terraform init terraform apply -auto-approve -var ami_id=$IMAGE_ID - public_ip=$(terraform output public_ip) + public_ip="$(terraform output public_ip_address | cut -d "\"" -f 2)" echo "PUBLIC_IP=$public_ip" >> $GITHUB_ENV - name: Test that the instance is running run: | - sh .github/scripts/check-instance.sh ${{ env.PUBLIC_IP }} + sh .github/scripts/check_instance.sh ${{ env.PUBLIC_IP }} - name: Test that Meilisearch is running correctly run: | sh .github/scripts/check_meilisearch_availability.sh ${{ env.PUBLIC_IP }} From 7af41609ebd0f3cf03b693ca81d16bbe011d5428 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Tue, 2 May 2023 13:03:33 +0200 Subject: [PATCH 3/9] Add logs to understand check instance script issue --- .github/scripts/check_instance.sh | 2 ++ .github/workflows/test.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/scripts/check_instance.sh b/.github/scripts/check_instance.sh index f829603..91790a3 100644 --- a/.github/scripts/check_instance.sh +++ b/.github/scripts/check_instance.sh @@ -1,6 +1,8 @@ #!/bin/bash PUBLIC_IP=$1 +echo "http://$PUBLIC_IP" +echo $PUBLIC_IP start_time=$(date +%s) while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' --max-time 5 http://$PUBLIC_IP)" != "200" ]]; do diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cdc9455..3cda1e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,6 +36,8 @@ jobs: - name: check env run: echo ${{ env.IMAGE_ID }} + - name: check env + run: echo $IMAGE_ID - name: Install Terraform run: | From c7e097bb6b9611383792bd593bda50c7246594e8 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Tue, 13 Jun 2023 14:08:52 +0200 Subject: [PATCH 4/9] Change the way packer is installed --- .github/workflows/test.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3cda1e0..59c82c5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,10 +20,9 @@ jobs: uses: actions/checkout@v2 - name: Install Packer - run: | - wget https://releases.hashicorp.com/packer/1.7.7/packer_1.7.7_linux_amd64.zip # change version - unzip packer_1.7.7_linux_amd64.zip - sudo mv packer /usr/local/bin/ + uses: hashicorp/setup-packer@main + with: + version: ${{ env.PACKER_VERSION }} - name: Build image run: | From f4da213816faee597420fadcb15664505deebc2b Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Tue, 13 Jun 2023 14:10:36 +0200 Subject: [PATCH 5/9] Remove unecessary echo's in test CI --- .github/workflows/test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59c82c5..b9b780a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,9 +28,6 @@ jobs: run: | packer init . image_id="$(packer build -var 'image_name=${{ github.run_id }}' -machine-readable -only "amazon-ebs.*" meilisearch.pkr.hcl | awk -F, '$0 ~/artifact,0,id/ {print $6}' | cut -d ":" -f 2)" - echo "hello" - echo "$image_id" - echo $image_id echo "IMAGE_ID=$image_id" >> $GITHUB_ENV - name: check env From c400f4cc679f5d43f76f260aef7f95848673080b Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Tue, 13 Jun 2023 15:29:01 +0200 Subject: [PATCH 6/9] Rollback github packer action --- .github/workflows/test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b9b780a..210c885 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,9 +20,10 @@ jobs: uses: actions/checkout@v2 - name: Install Packer - uses: hashicorp/setup-packer@main - with: - version: ${{ env.PACKER_VERSION }} + run: | + wget https://releases.hashicorp.com/packer/1.7.7/packer_1.7.7_linux_amd64.zip # change version + unzip packer_1.7.7_linux_amd64.zip + sudo mv packer /usr/local/bin/ - name: Build image run: | From a26449a3461543666b02586637594726feff87b3 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Tue, 13 Jun 2023 17:33:38 +0200 Subject: [PATCH 7/9] fix Required plugins are not installed during destroy of ami --- .github/scripts/check_meilisearch_availability.sh | 2 -- .github/workflows/test.yml | 7 ++++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/scripts/check_meilisearch_availability.sh b/.github/scripts/check_meilisearch_availability.sh index 02fbd30..bfaeecd 100644 --- a/.github/scripts/check_meilisearch_availability.sh +++ b/.github/scripts/check_meilisearch_availability.sh @@ -5,12 +5,10 @@ status="$(curl http://$PUBLIC_IP/health)" # Check if the variable content is equal to '{"status":"available"}' if [ "$status" != '{"status":"available"}' ]; then - # If not, echo the variable value and an error message, and exit with code 1 echo "Error: Meilisearch is not running correctly." echo "Server response: $status" exit 1 else - # If it is equal, echo a success message and exit with code 0 echo "Meilisearch is available." exit 0 fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 210c885..ce0569f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,9 +31,9 @@ jobs: image_id="$(packer build -var 'image_name=${{ github.run_id }}' -machine-readable -only "amazon-ebs.*" meilisearch.pkr.hcl | awk -F, '$0 ~/artifact,0,id/ {print $6}' | cut -d ":" -f 2)" echo "IMAGE_ID=$image_id" >> $GITHUB_ENV - - name: check env - run: echo ${{ env.IMAGE_ID }} - - name: check env + - name: check env.IMAGE_ID + run: echo "${{ env.IMAGE_ID }}" + - name: check IMAGE_ID run: echo $IMAGE_ID - name: Install Terraform @@ -59,6 +59,7 @@ jobs: - name: Destroy instance and AMI if: always() run: | + terraform init terraform destroy -var ami_id=$IMAGE_ID -auto-approve \ No newline at end of file From ed7b9ccbf65eb1d867f165b20c06daf2e088eb87 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Wed, 14 Jun 2023 12:24:55 +0200 Subject: [PATCH 8/9] Use packer github action with latest version of packer --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce0569f..bb2b403 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,15 +15,15 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + PACKER_VERSION: "latest" steps: - name: Checkout code uses: actions/checkout@v2 - name: Install Packer - run: | - wget https://releases.hashicorp.com/packer/1.7.7/packer_1.7.7_linux_amd64.zip # change version - unzip packer_1.7.7_linux_amd64.zip - sudo mv packer /usr/local/bin/ + uses: hashicorp/setup-packer@main + with: + version: ${{ env.PACKER_VERSION }} - name: Build image run: | From ccb9255ddfd96b48ac158013caabe3e0569dc006 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Wed, 14 Jun 2023 12:55:07 +0200 Subject: [PATCH 9/9] Add logs on build response --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bb2b403..7e2953b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,9 @@ jobs: - name: Build image run: | packer init . - image_id="$(packer build -var 'image_name=${{ github.run_id }}' -machine-readable -only "amazon-ebs.*" meilisearch.pkr.hcl | awk -F, '$0 ~/artifact,0,id/ {print $6}' | cut -d ":" -f 2)" + build_res="$(packer build -var 'image_name=${{ github.run_id }}' -machine-readable -only "amazon-ebs.*" meilisearch.pkr.hcl)" + echo $build_res + image_id="$(echo $build_res | awk -F, '$0 ~/artifact,0,id/ {print $6}' | cut -d ":" -f 2)" echo "IMAGE_ID=$image_id" >> $GITHUB_ENV - name: check env.IMAGE_ID