More terraform migration

Makefile

@@ -93,7 +93,7 @@ deploy_prod: check_account_prod
 	sam deploy $(common_params) --parameter-overrides $(run_env)=prod $(set_application_prefix)=$(application_key) $(set_application_name)="$(application_name)" S3BucketPrefix="$(s3_bucket_prefix)"
 	@echo "Deploying Terraform..."
 	$(eval MAIN_DISTRIBUTION_ID := $(shell aws cloudformation describe-stacks --stack-name $(application_key) --query "Stacks[0].Outputs[?OutputKey=='CloudfrontDistributionId'].OutputValue" --output text))
-	terraform -chdir=terraform/envs/prod init
+	terraform -chdir=terraform/envs/prod init -lockfile=readonly
 	terraform -chdir=terraform/envs/prod apply -auto-approve -var main_cloudfront_distribution_id="$(MAIN_DISTRIBUTION_ID)"
 	make postdeploy
 
@@ -102,7 +102,7 @@ deploy_dev: check_account_dev
 	sam deploy $(common_params) --parameter-overrides $(run_env)=dev $(set_application_prefix)=$(application_key) $(set_application_name)="$(application_name)" S3BucketPrefix="$(s3_bucket_prefix)"
 	@echo "Deploying Terraform..."
 	$(eval MAIN_DISTRIBUTION_ID := $(shell aws cloudformation describe-stacks --stack-name $(application_key) --query "Stacks[0].Outputs[?OutputKey=='CloudfrontDistributionId'].OutputValue" --output text))
-	terraform -chdir=terraform/envs/qa init
+	terraform -chdir=terraform/envs/qa init -lockfile=readonly
 	terraform -chdir=terraform/envs/qa apply -auto-approve -var main_cloudfront_distribution_id="$(MAIN_DISTRIBUTION_ID)"
 	make postdeploy
 
@@ -119,21 +119,25 @@ invalidate_cloudfront:
 	aws cloudfront wait invalidation-completed --distribution-id $(DISTRIBUTION_ID_2) --id $(INVALIDATION_ID_2)
 	@echo "CloudFront invalidation completed!"
 
+init_terraform:
+	terraform -chdir=terraform/envs/qa init
+	terraform -chdir=terraform/envs/prod init
+
 install:
 	yarn -D
 	pip install cfn-lint
-	terraform -chdir=terraform/envs/qa init
-	terraform -chdir=terraform/envs/prod init
 
 test_live_integration: install
 	yarn test:live
 
 test_unit: install
 	yarn lint
 	cfn-lint cloudformation/**/*
+	terraform -chdir=terraform/envs/qa init -reconfigure -backend=false -upgrade
 	terraform -chdir=terraform/envs/qa fmt -check
-	terraform -chdir=terraform/envs/prod fmt -check
 	terraform -chdir=terraform/envs/qa validate
+	terraform -chdir=terraform/envs/prod init -reconfigure -backend=false
+	terraform -chdir=terraform/envs/prod fmt -check
 	terraform -chdir=terraform/envs/prod validate
 	yarn prettier
 	yarn test:unit
@@ -149,3 +153,7 @@ dev_health_check:
 
 prod_health_check:
 	curl -f https://core.acm.illinois.edu/api/v1/healthz && curl -f https://core.acm.illinois.edu
+
+lock_terraform: init_terraform
+	terraform -chdir=terraform/envs/qa providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=darwin_arm64 -platform=linux_amd64 -platform=linux_arm64
+	terraform -chdir=terraform/envs/prod providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=darwin_arm64 -platform=linux_amd64 -platform=linux_arm64

cloudformation/main.yml

@@ -22,10 +22,6 @@ Parameters:
     Description: How long the SQS lambda is permitted to run (in seconds)
     Default: 180
     Type: Number
-  SqsMessageTimeout:
-    Description: MessageVisibilityTimeout for the SQS Lambda queue (should be at least (numMaxRetry + 1)*SqsLambdaTimeout)
-    Default: 720
-    Type: Number
   S3BucketPrefix:
     Description: S3 bucket prefix which will ensure global uniqueness
     Type: String
@@ -86,22 +82,14 @@ Resources:
         RunEnvironment: !Ref RunEnvironment
         LambdaFunctionName: !Sub ${ApplicationPrefix}-lambda
         SesEmailDomain: !FindInMap [General, !Ref RunEnvironment, SesDomain]
-        SqsQueueArn: !GetAtt AppSQSQueues.Outputs.MainQueueArn
+        SqsQueueArn: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs"
         LinkryKvArn: !GetAtt LinkryRecordsCloudfrontStore.Arn
 
   AppLogGroups:
     Type: AWS::Serverless::Application
     Properties:
       Location: ./logs.yml
 
-  AppSQSQueues:
-    Type: AWS::Serverless::Application
-    Properties:
-      Location: ./sqs.yml
-      Parameters:
-        QueueName: !Sub ${ApplicationPrefix}-sqs
-        MessageTimeout: !Ref SqsMessageTimeout
-
   LinkryRecordSetv4:
     Condition: IsDev
     Type: AWS::Route53::RecordSet
@@ -319,7 +307,7 @@ Resources:
       - AppSqsLambdaFunction
     Properties:
       BatchSize: 5
-      EventSourceArn: !GetAtt AppSQSQueues.Outputs.MainQueueArn
+      EventSourceArn: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs"
       FunctionName: !Sub ${ApplicationPrefix}-sqs-lambda
       FunctionResponseTypes:
         - ReportBatchItemFailures
@@ -330,7 +318,7 @@ Resources:
       - AppSqsLambdaFunction
     Properties:
       BatchSize: 5
-      EventSourceArn: !GetAtt AppSQSQueues.Outputs.SalesEmailQueueArn
+      EventSourceArn: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs-sales"
       FunctionName: !Sub ${ApplicationPrefix}-sqs-lambda
       FunctionResponseTypes:
         - ReportBatchItemFailures
@@ -1011,4 +999,4 @@ Outputs:
 
   SalesEmailQueueArn:
     Description: Sales Email Queue Arn
-    Value: !GetAtt AppSQSQueues.Outputs.SalesEmailQueueArn
+    Value: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs-sales"

cspell.config.yaml

@@ -6,6 +6,7 @@ ignorePaths:
 dictionaryDefinitions: []
 dictionaries: []
 words:
+  - redrive
   - UIUC
 ignoreWords: []
 import: []

terraform/envs/prod/main.tf

@@ -1,3 +1,8 @@
+data "aws_caller_identity" "current" {}
+locals {
+  account_id = data.aws_caller_identity.current.account_id
+}
+
 terraform {
   required_providers {
     aws = {
@@ -7,6 +12,12 @@ terraform {
   }
 
   required_version = ">= 1.2"
+  backend "s3" {
+    bucket       = "298118738376-terraform"
+    key          = "infra-core-api"
+    region       = "us-east-1"
+    use_lockfile = true
+  }
 }
 
 provider "aws" {
@@ -17,7 +28,6 @@ provider "aws" {
     }
   }
 }
-
 import {
   to = aws_cloudwatch_log_group.main_app_logs
   id = "/aws/lambda/${var.ProjectId}-lambda"
@@ -34,3 +44,8 @@ module "app_alarms" {
   priority_sns_arn                = var.GeneralSNSAlertArn
   standard_sns_arn                = var.PrioritySNSAlertArn
 }
+
+module "sqs_queues" {
+  source          = "../../modules/sqs"
+  resource_prefix = var.ProjectId
+}

terraform/envs/qa/main.tf

@@ -7,6 +7,13 @@ terraform {
   }
 
   required_version = ">= 1.2"
+
+  backend "s3" {
+    bucket       = "427040638965-terraform"
+    key          = "infra-core-api"
+    region       = "us-east-1"
+    use_lockfile = true
+  }
 }
 
 provider "aws" {
@@ -17,7 +24,15 @@ provider "aws" {
     }
   }
 }
+import {
+  to = aws_cloudwatch_log_group.main_app_logs
+  id = "/aws/lambda/${var.ProjectId}-lambda"
+}
 resource "aws_cloudwatch_log_group" "main_app_logs" {
   name              = "/aws/lambda/${var.ProjectId}-lambda"
   retention_in_days = var.LogRetentionDays
 }
+module "sqs_queues" {
+  source          = "../../modules/sqs"
+  resource_prefix = var.ProjectId
+}

terraform/modules/sqs/main.tf

@@ -0,0 +1,46 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}
+
+resource "aws_sqs_queue" "app_dlq" {
+  name                       = "${var.resource_prefix}-sqs-dlq"
+  visibility_timeout_seconds = var.sqs_message_timeout
+  message_retention_seconds  = 1209600
+}
+
+resource "aws_sqs_queue" "app_queue" {
+  name                       = "${var.resource_prefix}-sqs"
+  visibility_timeout_seconds = var.sqs_message_timeout
+  redrive_policy = jsonencode({
+    deadLetterTargetArn = aws_sqs_queue.app_dlq.arn
+    maxReceiveCount     = 3
+  })
+}
+
+resource "aws_sqs_queue" "sales_email_queue" {
+  name                       = "${var.resource_prefix}-sqs-sales"
+  visibility_timeout_seconds = var.sqs_message_timeout
+  redrive_policy = jsonencode({
+    deadLetterTargetArn = aws_sqs_queue.app_dlq.arn
+    maxReceiveCount     = 3
+  })
+}
+
+output "main_queue_arn" {
+  description = "Main Queue Arn"
+  value       = aws_sqs_queue.app_queue.arn
+}
+
+output "dlq_arn" {
+  description = "Dead-letter Queue Arn"
+  value       = aws_sqs_queue.app_dlq.arn
+}
+
+output "sales_email_queue_arn" {
+  description = "Sales Email Queue Arn"
+  value       = aws_sqs_queue.sales_email_queue.arn
+}

terraform/modules/sqs/variables.tf

@@ -0,0 +1,16 @@
+variable "resource_prefix" {
+  type        = string
+  description = "Prefix before each resource"
+}
+
+variable "sqs_message_timeout" {
+  type        = number
+  description = "SQS Message timeout in seconds"
+  default     = 720
+}
+
+variable "dlq_message_retention" {
+  type        = number
+  description = "DLQ Message retention in seconds"
+  default     = 1209600
+}