From 0f74db8ce0986623a5e5ae91c47b46afd299f3d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 10:12:10 +0000 Subject: [PATCH 01/42] Bump hashicorp/setup-terraform from 1 to 2 Bumps [hashicorp/setup-terraform](https://github.com/hashicorp/setup-terraform) from 1 to 2. - [Release notes](https://github.com/hashicorp/setup-terraform/releases) - [Changelog](https://github.com/hashicorp/setup-terraform/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/setup-terraform/compare/v1...v2) --- updated-dependencies: - dependency-name: hashicorp/setup-terraform dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6c37b2..76f6dc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 - name: Terraform Setup - uses: hashicorp/setup-terraform@v1 + uses: hashicorp/setup-terraform@v2 with: terraform_version: '0.15.x' @@ -41,7 +41,7 @@ jobs: aws-region: us-west-2 - name: Terraform Setup - uses: hashicorp/setup-terraform@v1 + uses: hashicorp/setup-terraform@v2 with: terraform_version: ${{ matrix.terraform }} From d92e34655e0efb532a92fe389aadf9e6bba028ca Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Mon, 16 May 2022 06:44:05 -0600 Subject: [PATCH 02/42] saving work --- README.md | 9 +- cloudformation.json | 582 ++++++++++++++++++++++++++++++++++++++++++++ lambda/function.zip | Bin 1167 -> 0 bytes lambda/index.js | 86 ------- main.tf | 190 ++++++++------- outputs.tf | 54 +++- variables.tf | 92 +++---- 7 files changed, 795 insertions(+), 218 deletions(-) create mode 100644 cloudformation.json delete mode 100644 lambda/function.zip delete mode 100644 lambda/index.js diff --git a/README.md b/README.md index 1c08f87..2ac6456 100644 --- a/README.md +++ b/README.md @@ -79,4 +79,11 @@ resources that the for_each depends on. ## Deployment -If you update the Lambda function code, be sure to run `zip -r function.zip .` in the `lambda` folder. \ No newline at end of file +If you update the Lambda function code, be sure to run `zip -r function.zip .` in the `lambda` folder. + +## Development + +To update to the latest bundle of Cloudwatch2Humio: +1. Clone the git repository: [https://github.com/humio/cloudwatch2humio](https://github.com/humio/cloudwatch2humio) +4. Copy the `$PROJECT/cloudformation.json` file to this project replacing the old cloudformation template. +5. Ensure that the necessary refactors are implemented in terraform. diff --git a/cloudformation.json b/cloudformation.json new file mode 100644 index 0000000..b67a115 --- /dev/null +++ b/cloudformation.json @@ -0,0 +1,582 @@ +{ + "AWSTemplateFormatVersion" : "2010-09-09", + "Description" : "CloudWatch to Humio Integration for sending CloudWatch Logs and Metrics to Humio.", + "Parameters" : { + "HumioProtocol" : { + "Type" : "String", + "Description" : "The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended.", + "Default" : "https" + }, + "HumioHost" : { + "Type" : "String", + "Description" : "The host to ship Humio log/metric events to.", + "Default" : "cloud.humio.com" + }, + "HumioIngestToken" : { + "Type" : "String", + "Description" : "The value of the ingest token for the repository from your Humio account to ship log/metric events to.", + "Default" : "", + "NoEcho" : true + }, + "HumioLambdaLogRetention" : { + "Type" : "Number", + "Description" : "Number of days to retain CloudWatch logs from the Humio Lambda functions.", + "Default": 1 + }, + "EnableCloudWatchLogsAutoSubscription" : { + "Type" : "String", + "AllowedValues" : [ + "true", + "false" + ], + "Description" : "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable.", + "Default" : "true" + }, + "HumioCloudWatchLogsSubscriptionPrefix" : { + "Type" : "String", + "Description" : "Humio will only subscribe to log groups with the prefix specified.", + "Default" : "" + }, + "EnableCloudWatchLogsBackfillerAutoRun" : { + "Type" : "String", + "AllowedValues" : [ + "true", + "false" + ], + "Description" : "Make the backfiller run automatically when created. Set to 'true' to enable.", + "Default" : "true" + }, + "EnableVPCForIngesterLambdas" : { + "Type" : "String", + "AllowedValues" : [ + "true", + "false" + ], + "Description" : "Use a VPC for the lambda ingester functions. Set to 'true' to enable.", + "Default" : "false" + }, + "SecurityGroupIds" : { + "Type" : "CommaDelimitedList", + "Description" : "A comma separated list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled." + }, + "SubnetIds" : { + "Type" : "CommaDelimitedList", + "Description" : "A comma separated list of subnet ids used by the VPC configuration that the ingester lamda functions will be deployed into. Only required if VPC is enabled." + }, + "HumioLambdaLogLevel" : { + "Type" : "String", + "AllowedValues" : ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + "Default" : "INFO" + }, + "Version" : { + "Type" : "String", + "Description" : "The version of the integration you want installed.", + "Default" : "v1.2.1" + } + }, + "Conditions" : { + "CreateAutoSubscriptionResources" : { + "Fn::Equals" : [ { "Ref" : "EnableCloudWatchLogsAutoSubscription" }, "true" ] + }, + "CreateHumioBackfillerAutoRunner" : { + "Fn::Equals" : [ { "Ref" : "EnableCloudWatchLogsBackfillerAutoRun" }, "true" ] + }, + "ConfigureVPCForIngesterLambdas" : { + "Fn::Equals" : [ { "Ref" : "EnableVPCForIngesterLambdas" }, "true" ] + } + }, + "Resources" : { + "HumioCloudWatchRole" : { + "Type" : "AWS::IAM::Role", + "Properties" : { + "AssumeRolePolicyDocument" : { + "Version" : "2012-10-17", + "Statement" : [ + { + "Action" : "sts:AssumeRole", + "Principal" : { + "Service" : [ + "lambda.amazonaws.com", + "apigateway.amazonaws.com", + "logs.amazonaws.com" + ] + }, + "Effect" : "Allow", + "Sid" : "" + } + ] + }, + "Policies" : [ + { + "PolicyName" : "humio_cloudwatch_role", + "PolicyDocument" : { + "Fn::If" : [ "ConfigureVPCForIngesterLambdas", + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "lambda:GetFunction", + "lambda:InvokeFunction", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:DescribeSubscriptionFilters", + "logs:PutSubscriptionFilter", + "logs:DeleteSubscriptionFilter", + "logs:PutLogEvents", + "logs:GetLogEvents", + "logs:FilterLogEvents", + "cloudwatch:GetMetricData", + "cloudwatch:GetMetricStatistics", + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:CreateNetworkInterfacePermission", + "ec2:DeleteNetworkInterface" + ], + "Resource" : "*" + } + ] + }, + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "lambda:GetFunction", + "lambda:InvokeFunction", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:DescribeSubscriptionFilters", + "logs:PutSubscriptionFilter", + "logs:DeleteSubscriptionFilter", + "logs:PutLogEvents", + "logs:GetLogEvents", + "logs:FilterLogEvents", + "cloudwatch:GetMetricData", + "cloudwatch:GetMetricStatistics" + ], + "Resource" : "*" + } + ] + } + ] + } + } + ] + } + }, + "HumioCloudWatchLogsIngester" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code" : { + "S3Bucket" : { + "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] + }, + "S3Key" : { + "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] + } + }, + "Environment" : { + "Variables" : { + "humio_protocol" : { "Ref" : "HumioProtocol" }, + "humio_host" : { "Ref" : "HumioHost" }, + "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, + "log_level" : { "Ref" : "HumioLambdaLogLevel" } + } + }, + "VpcConfig" : { + "Fn::If" : [ "ConfigureVPCForIngesterLambdas", + { + "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, + "SubnetIds" : { "Ref" : "SubnetIds" } + }, + { + "Ref" : "AWS::NoValue" + } + ] + }, + "Description" : "CloudWatch Logs to Humio ingester.", + "Handler" : "logs_ingester.lambda_handler", + "MemorySize" : "128", + "Role" : { + "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + }, + "Runtime" : "python3.8", + "Timeout" : "300" + } + }, + "HumioCloudWatchLogsIngesterPermission" : { + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] + }, + "Principal" : "logs.amazonaws.com" + } + }, + "HumioCloudWatchLogsIngesterLogGroup" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsIngester" } ] ] + }, + "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + } + }, + "HumioCloudWatchLogsSubscriber" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code" : { + "S3Bucket" : { + "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] + }, + "S3Key" : { + "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] + } + }, + "Environment" : { + "Variables" : { + "humio_log_ingester_arn" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] + }, + "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }, + "log_level" : { "Ref" : "HumioLambdaLogLevel" } + } + }, + "Description" : "CloudWatch Logs to Humio log group subscriber.", + "Handler" : "logs_subscriber.lambda_handler", + "MemorySize" : "128", + "Role" : { + "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + }, + "Runtime" : "python3.8", + "Timeout" : "300" + } + }, + "HumioCloudWatchLogsSubscriberPermission" : { + "Condition" : "CreateAutoSubscriptionResources", + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] + }, + "Principal" : "events.amazonaws.com", + "SourceAccount" : { "Ref" : "AWS::AccountId" } + } + }, + "HumioCloudWatchLogsSubscriberPermission2" : { + "Condition" : "CreateAutoSubscriptionResources", + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] + }, + "Principal" : "events.amazonaws.com", + "SourceArn" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberEventRule", "Arn" ] + } + } + }, + "HumioCloudWatchLogsSubscriberLogGroup" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsSubscriber" } ] ] + }, + "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + } + }, + "HumioCloudWatchLogsBackfiller" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code" : { + "S3Bucket" : { + "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] + }, + "S3Key" : { + "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] + } + }, + "Environment" : { + "Variables" : { + "humio_log_ingester_arn" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] + }, + "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }, + "humio_protocol" : { "Ref" : "HumioProtocol" }, + "humio_host" : { "Ref" : "HumioHost" }, + "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, + "log_level" : { "Ref" : "HumioLambdaLogLevel" } + } + }, + "Description" : "CloudWatch Logs to Humio logs backfiller.", + "Handler" : "logs_backfiller.lambda_handler", + "MemorySize" : "128", + "Role" : { + "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + }, + "Runtime" : "python3.8", + "Timeout" : "300" + } + }, + "HumioCloudWatchLogsBackfillerPermission" : { + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsBackfiller", "Arn" ] + }, + "Principal" : "logs.amazonaws.com" + } + }, + "HumioCloudWatchLogsBackfillerLogGroup" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsBackfiller" } ] ] + }, + "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + } + }, + "HumioBackfillerAutoRunner" : { + "Condition" : "CreateHumioBackfillerAutoRunner", + "DependsOn" : [ "HumioCloudWatchLogsBackfiller" ], + "Type" : "Custom::BackfillerAutoRunner", + "Properties" : { + "ServiceToken" : { "Fn::GetAtt" : [ "HumioCloudWatchLogsBackfiller", "Arn" ] }, + "StackName" : { "Ref" : "AWS::StackName" } + } + }, + "HumioCloudWatchLogsSubscriberS3Bucket" : { + "Condition" : "CreateAutoSubscriptionResources", + "Type" : "AWS::S3::Bucket", + "Properties" : { + "AccessControl" : "BucketOwnerFullControl", + "BucketName" : { + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", "cloudtrail" ] ] + } + } + }, + "HumioCloudWatchLogsSubscriberS3BucketPolicy" : { + "Condition" : "CreateAutoSubscriptionResources", + "DependsOn" : [ "HumioCloudWatchLogsSubscriberS3Bucket" ], + "Type" : "AWS::S3::BucketPolicy", + "Properties" : { + "Bucket" : { "Ref" : "HumioCloudWatchLogsSubscriberS3Bucket" }, + "PolicyDocument" : { + "Version" : "2012-10-17", + "Statement" : [ + { + "Sid" : "AWSCloudTrailAclCheck20150319", + "Effect" : "Allow", + "Principal" : { + "Service" : "cloudtrail.amazonaws.com" + }, + "Action" : "s3:GetBucketAcl", + "Resource" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberS3Bucket", "Arn" ] + } + }, + { + "Sid" : "AWSCloudTrailWrite20150319", + "Effect" : "Allow", + "Principal" : { + "Service" : "cloudtrail.amazonaws.com" + }, + "Action" : "s3:PutObject", + "Resource" : { + "Fn::Join" : [ "", [ { "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberS3Bucket", "Arn" ] }, "/AWSLogs/", { "Ref" : "AWS::AccountId" }, "/*" ] ] + }, + "Condition" : { + "StringEquals" : { "s3:x-amz-acl" : "bucket-owner-full-control" } + } + } + ] + } + } + }, + "HumioCloudWatchLogsSubscriberCloudTrail" : { + "Condition" : "CreateAutoSubscriptionResources", + "DependsOn" : [ "HumioCloudWatchLogsSubscriberS3BucketPolicy" ], + "Type" : "AWS::CloudTrail::Trail", + "Properties" : { + "EnableLogFileValidation" : false, + "IncludeGlobalServiceEvents" : true, + "IsMultiRegionTrail" : true, + "IsLogging" : true, + "S3BucketName" : { + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", "cloudtrail" ] ] + }, + "TrailName" : { + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", { "Ref" : "AWS::AccountId" } ] ] + } + } + }, + "HumioCloudWatchLogsSubscriberEventRule" : { + "Condition" : "CreateAutoSubscriptionResources", + "DependsOn" : [ "HumioCloudWatchLogsSubscriber" ], + "Type" : "AWS::Events::Rule", + "Properties" : { + "Description" : "Humio log group auto subscription event rule.", + "EventPattern" : { + "source" : [ "aws.logs" ], + "detail-type" : [ "AWS API Call via CloudTrail" ], + "detail" : { + "eventSource" : [ "logs.amazonaws.com" ], + "eventName" : [ "CreateLogGroup" ] + } + }, + "Name" : { + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio-auto-subscription-rule" ] ] + }, + "Targets" : [ + { + "Id" : { + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio-auto-subscription-rule" ] ] + }, + "Arn" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] + } + } + ] + } + }, + "HumioCloudWatchMetricIngester" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code" : { + "S3Bucket" : { + "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] + }, + "S3Key" : { + "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] + } + }, + "Environment" : { + "Variables" : { + "humio_protocol" : { "Ref" : "HumioProtocol" }, + "humio_host" : { "Ref" : "HumioHost" }, + "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, + "log_level" : { "Ref" : "HumioLambdaLogLevel" } + } + }, + "VpcConfig" : { + "Fn::If" : [ "ConfigureVPCForIngesterLambdas", + { + "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, + "SubnetIds" : { "Ref" : "SubnetIds" } + }, + { + "Ref" : "AWS::NoValue" + } + ] + }, + "Description" : "CloudWatch Metrics to Humio ingester.", + "Handler" : "metric_ingester.lambda_handler", + "MemorySize" : "128", + "Role" : { + "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + }, + "Runtime" : "python3.8", + "Timeout" : "300" + } + }, + "HumioCloudWatchMetricIngesterPermission" : { + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchMetricIngester", "Arn" ] + }, + "Principal" : "logs.amazonaws.com" + } + }, + "HumioCloudWatchMetricIngesterLogGroup" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchMetricIngester" } ] ] + }, + "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + } + }, + "HumioCloudWatchMetricStatisticsIngester" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code" : { + "S3Bucket" : { + "Fn::Join" : [ "-", [ "humio-public", { "Ref" : "AWS::Region" } ] ] + }, + "S3Key" : { + "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] + } + }, + "Environment" : { + "Variables" : { + "humio_protocol" : { "Ref" : "HumioProtocol" }, + "humio_host" : { "Ref" : "HumioHost" }, + "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, + "log_level" : { "Ref" : "HumioLambdaLogLevel" } + } + }, + "VpcConfig" : { + "Fn::If" : [ "ConfigureVPCForIngesterLambdas", + { + "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, + "SubnetIds" : { "Ref" : "SubnetIds" } + }, + { + "Ref" : "AWS::NoValue" + } + ] + }, + "Description" : "CloudWatch Metrics Statistics to Humio ingester.", + "Handler" : "metric_statistics_ingester.lambda_handler", + "MemorySize" : "128", + "Role" : { + "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + }, + "Runtime" : "python3.8", + "Timeout" : "300" + } + }, + "HumioCloudWatchMetricStatisticsIngesterPermission" : { + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchMetricStatisticsIngester", "Arn" ] + }, + "Principal" : "logs.amazonaws.com" + } + }, + "HumioCloudWatchMetricStatisticsIngesterLogGroup" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchMetricStatisticsIngester" } ] ] + }, + "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + } + } + } +} diff --git a/lambda/function.zip b/lambda/function.zip deleted file mode 100644 index c8abc7f80ff469a6ee6c344c549478e11d81c9e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1167 zcmWIWW@Zs#U|`^2&@8hF4n43e<|Q)&!$%H~hzvt!UP@|(URH5v2qy#cx%%M5tzE&1 zr4`%^j4ZF1nHg9ZnV1=R!*cs?xe5F`w|+*q*3Ga)&)dQF3KL)HwM*SPpy;^DE3hQ; zjp4l$6?4Bz^>zQbZ@v3<_$;rd+~3Z;*<Dm#twAuL5Jz7K#})Gi`pc*G9*0#>*u2`%~Q&5Bf`cGTe7#I42#&xifp~5~c+A zdB4NhKbv2>^JmMhE$4fg*KHO09kjUc)A4VGXN%Txm9*a!>~4}bXm}}NrXqfmk&83E z#UpKATN7V8yI{OS#)Wi2rR=}jSC*ctV$@`G@oAt5wYVny5dG_p9))M^m zk~vL1B<}ty?;5iX$D_vAG>16 zHJDZLdg7zxg4RVo6FYUL?fV`!Et6&Dm4v`84OPhp5==@$lP6Ej%u=0doUqZuRO|nd z;LF=*OIIl*F?2}$k-nZiEvf8+=j3L;DK>|lG*|u!&(2i#%PH9Bzwy~|?Z^A1wyx>i z@yvJkZLweS+5LYmG=HgA46~fFQ8D4GrbMjsX4N3+Uo&p>7#RL{oIdFc?^j;V*e5Px zR*&748t?P(iTKa@`KQdD7}q$4{f+9a^HkEDZu|=}>ha=Fvwf*@D0p>Y=Ch!el6UU? z3)){a_d=)U`J-RHT%J;-;^lWYk!d;4%20``>HRB&B`5xTR=9h0D(5GTP=4b@j>|Z^ zPQIJ8*!`Hz`QSwxR~GzQy5OW_eDuEUzN*pO{n>@dCE-Cns&Q{BKb7^zEM(%Zllk)G zZu`|wA2WCUtKE3l_O0Cim^JRaPjg;NpL-~2<|W6BmxM$|!x391N-B|dsW^&KtchCFt z&fd+P=J$!S?uS`P@WB*?=$%hPYWZ$`Wb(;;_0M+h7Fk*5v-5NoXgD)R$1LONNoTt1 zaDiX!OVCz{pNA#a>6G_=)1Ex@{lb$o_x$*#_+tIts$l7#t1JZOg}lz&ezUAJO4hif z+;rFXd`IeYg>?{;(RxUuU*pGDL8G!w3AA9TDr6F3Q<&&CUTFyGNMd{A;mi2q<)22-ipS{#B=h|I~I=-sp`JcVDDqR-xh`u{*;dV4> zQF<-o$Kph*V?MW%KYZJBAU=QZg2kJ6M&Dd`CeG#E^>e?@sARcI1lcRkj&*iC9M~cq zrJ}Yx;le7pBaZ53p_^xPZ)Wf#pVf!``8(@&_o^(5&-?-2j7)OOxC#piU?IZ5zzD=k8bK^X b(ZLESI?#%Z0B=?{kUB;n^aIk`EFc~L!>$%5 diff --git a/lambda/index.js b/lambda/index.js deleted file mode 100644 index 9bbcd72..0000000 --- a/lambda/index.js +++ /dev/null @@ -1,86 +0,0 @@ -const zlib = require('zlib') -const https = require('https') - -function sendLogEventsToHumio (logEvents) { - const events = logEvents - .map(event => { - try { - const attributes = JSON.parse(event.message) // Parsing so we don't stringify it twice. - return { - timestamp: event.timestamp, - timezone: 'America/Denver', - attributes: { - ...attributes, - SubIdxNM: process.env.SUB_IDX_NM - } - } - } catch(e) { - console.info('Could not parse JSON', e) - } - return null - }) - .filter(val => !!val) - - if (!events.length) { - throw Error("No JSON-parseable events") - } - - const data = JSON.stringify([{ - tags: { - SubIdxNM: process.env.SUB_IDX_NM - }, - events - }]) - console.info('Sending data.', data) - - const options = { - hostname: (process.env.ENV === 'prd') ? 'oit-humio.byu.edu' : 'oit-humio-dev.byu.edu', - port: 443, - path: '/api/v1/ingest/humio-structured', - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': data.length, - Authorization: 'Bearer ' + process.env.HUMIO_INGEST_TOKEN - } - } - - return new Promise((resolve, reject) => { - const req = https.request(options, function (res) { - if (res.statusCode >= 400) { - reject(new Error(`[Humio API Error] ${res.statusCode} - ${res.statusMessage}`)) - } else { - console.log('Send logs successful. Code: ', res.statusCode) - resolve() - } - }) - - req.on('error', (error) => { - const errorMsg = `[HTTPS Error] ${error.name} - ${error.message}` - console.error(errorMsg) - reject(errorMsg) - }) - - req.write(data) - req.end() - }) -} - -exports.handler = function (event, context) { - console.debug('Event: ' + JSON.stringify(event, null, 2)) - - const payload = new Buffer.from(event.awslogs.data, 'base64') - return zlib.gunzip(payload, function (e, decodedEvent) { - if (e !== null) { - console.error(e) - context.fail(e) - } else { - console.debug('Decoded event: ' + decodedEvent) - decodedEvent = JSON.parse(decodedEvent.toString('ascii')) - sendLogEventsToHumio(decodedEvent.logEvents).catch(e => { - console.error(e) - context.fail(e) - }) - } - }) -} diff --git a/main.tf b/main.tf index 3640ee5..7b75c8f 100644 --- a/main.tf +++ b/main.tf @@ -5,95 +5,119 @@ terraform { } } -locals { - filename = "${path.module}/lambda/function.zip" +module "acs" { + source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.4.1" } -resource "aws_lambda_function" "logging" { - function_name = "${var.app_name}-logs-to-humio" - filename = local.filename - handler = "index.handler" - runtime = "nodejs14.x" - role = aws_iam_role.logging.arn - source_code_hash = filebase64sha256(local.filename) - timeout = var.timeout - memory_size = var.memory_size - tags = var.tags - - environment { - variables = { - ENV = var.app_env - HUMIO_INGEST_TOKEN = var.humio_ingest_token - SUB_IDX_NM = var.sub_idx_nm - } - } - - vpc_config { - security_group_ids = [aws_security_group.logging.id] - subnet_ids = var.private_vpn_subnet_ids - } +data "local_file" "cloudformation" { + filename = "${path.module}/cloudformation.json" } -resource "aws_security_group" "logging" { - name = "${var.app_name}-logs-to-humio" - vpc_id = var.vpn_vpc_id +resource "aws_cloudformation_stack" "cloudwatch" { + name = "${var.app_name}-cloudwatch2humio" - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] + parameters = { + HumioProtocol = var.humio_protocol + HumioHost = var.humio_host + HumioIngestToken = var.humio_ingest_token + HumioLambdaLogRetention = var.humio_lambda_log_retention + EnableCloudWatchLogsAutoSubscription = tostring(var.enable_cloudwatch_logs_auto_subscription) + HumioCloudWatchLogsSubscriptionPrefix = var.humio_cloudwatch_logs_subscription_prefix + EnableCloudWatchLogsBackfillerAutoRun = tostring(var.enable_cloudwatch_logs_backfiller_autorun) + EnableVPCForIngesterLambdas = tostring(var.enable_vpc_for_ingester_lambdas) + SecurityGroupIds = var.security_group_ids + SubnetIds = var.subnet_ids + HumioLambdaLogLevel = var.humio_lambda_log_level } -} - -resource "aws_iam_role" "logging" { - name = "${var.app_name}-logs-to-humio-role" - permissions_boundary = var.role_permissions_boundary - assume_role_policy = jsonencode({ - Version = "2012-10-17", - Statement = [ - { - Action = "sts:AssumeRole", - Principal = { - Service = "lambda.amazonaws.com" - }, - Effect = "Allow", - Sid = "" - } - ] - }) - tags = var.tags -} - -resource "aws_iam_role_policy_attachment" "basic_execution" { - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - role = aws_iam_role.logging.name -} - -resource "aws_iam_role_policy_attachment" "logging_eni_attach" { - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaENIManagementAccess" - role = aws_iam_role.logging.name -} - -resource "aws_lambda_permission" "logging" { - for_each = var.log_group_arns - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.logging.function_name - principal = "logs.amazonaws.com" - source_arn = "${each.key}:*" -} -resource "aws_cloudwatch_log_subscription_filter" "logging" { - for_each = var.log_group_names - depends_on = [aws_lambda_permission.logging] - destination_arn = aws_lambda_function.logging.arn - filter_pattern = var.filter_pattern - log_group_name = each.key - name = "logging_default" + template_body = data.local_file.cloudformation.content } -resource "aws_cloudwatch_log_group" "logging" { - name = "/aws/lambda/${aws_lambda_function.logging.function_name}" - retention_in_days = var.log_retention_in_days -} \ No newline at end of file +#resource "aws_lambda_function" "logging" { +# function_name = "${var.app_name}-logs-to-humio" +# filename = local.filename +# handler = "index.handler" +# runtime = "nodejs14.x" +# role = aws_iam_role.logging.arn +# source_code_hash = filebase64sha256(local.filename) +# timeout = var.timeout +# memory_size = var.memory_size +# tags = var.tags +# +# environment { +# variables = { +# ENV = var.app_env +# HUMIO_INGEST_TOKEN = var.humio_ingest_token +# SUB_IDX_NM = var.sub_idx_nm +# } +# } +# +# vpc_config { +# security_group_ids = [aws_security_group.logging.id] +# subnet_ids = var.private_vpn_subnet_ids +# } +#} +# +#resource "aws_security_group" "logging" { +# name = "${var.app_name}-logs-to-humio" +# vpc_id = var.vpn_vpc_id +# +# egress { +# from_port = 0 +# to_port = 0 +# protocol = "-1" +# cidr_blocks = ["0.0.0.0/0"] +# ipv6_cidr_blocks = ["::/0"] +# } +#} +# +#resource "aws_iam_role" "logging" { +# name = "${var.app_name}-logs-to-humio-role" +# permissions_boundary = var.role_permissions_boundary +# assume_role_policy = jsonencode({ +# Version = "2012-10-17", +# Statement = [ +# { +# Action = "sts:AssumeRole", +# Principal = { +# Service = "lambda.amazonaws.com" +# }, +# Effect = "Allow", +# Sid = "" +# } +# ] +# }) +# tags = var.tags +#} +# +#resource "aws_iam_role_policy_attachment" "basic_execution" { +# policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +# role = aws_iam_role.logging.name +#} +# +#resource "aws_iam_role_policy_attachment" "logging_eni_attach" { +# policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaENIManagementAccess" +# role = aws_iam_role.logging.name +#} +# +#resource "aws_lambda_permission" "logging" { +# for_each = var.log_group_arns +# action = "lambda:InvokeFunction" +# function_name = aws_lambda_function.logging.function_name +# principal = "logs.amazonaws.com" +# source_arn = "${each.key}:*" +#} +# +#resource "aws_cloudwatch_log_subscription_filter" "logging" { +# for_each = var.log_group_names +# depends_on = [aws_lambda_permission.logging] +# destination_arn = aws_lambda_function.logging.arn +# filter_pattern = var.filter_pattern +# log_group_name = each.key +# name = "logging_default" +#} +# +#resource "aws_cloudwatch_log_group" "logging" { +# name = "/aws/lambda/${aws_lambda_function.logging.function_name}" +# retention_in_days = var.log_retention_in_days +#} diff --git a/outputs.tf b/outputs.tf index d9096f0..4e5a0cf 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,3 +1,51 @@ -output "function_name" { - value = aws_lambda_function.logging.function_name -} \ No newline at end of file +output "app_env" { + value = var.app_env +} + +output "app_name" { + value = var.app_name +} + +output "humio_protocol" { + value = var.humio_protocol +} + +output "humio_host" { + value = var.humio_host +} + +output "humio_ingest_token" { + value = var.humio_ingest_token +} + +output "humio_lambda_log_retention" { + value = var.humio_lambda_log_retention +} + +output "enable_cloudwatch_logs_auto_subscription" { + value = var.enable_cloudwatch_logs_auto_subscription +} + +output "humio_cloudwatch_logs_subscription_prefix" { + value = var.humio_cloudwatch_logs_subscription_prefix +} + +output "enable_cloudwatch_logs_backfiller_autorun" { + value = var.enable_cloudwatch_logs_backfiller_autorun +} + +output "enable_vpc_for_ingester_lambdas" { + value = var.enable_vpc_for_ingester_lambdas +} + +output "security_group_ids" { + value = var.security_group_ids +} + +output "subnet_ids" { + value = var.subnet_ids +} + +output "humio_lambda_log_level" { + value = var.humio_lambda_log_level +} diff --git a/variables.tf b/variables.tf index 353e7c1..280a35b 100644 --- a/variables.tf +++ b/variables.tf @@ -8,69 +8,71 @@ variable "app_name" { description = "The application name to include in the name of resources created." } -// TODO Move to ACS as SSM parameter -variable "humio_ingest_token" { - type = string - description = "The ingest token for Humio logs." - sensitive = true +variable "humio_protocol" { + type = string + description = "The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended." + default = "https" } -variable "log_group_arns" { - type = set(string) - description = "The ARNs of CloudWatch log groups that should be forwarded to Humio." +variable "humio_host" { + type = string + description = "The host to ship Humio log/metric events to." } -variable "log_group_names" { - type = set(string) - description = "The names of the CloudWatch log groups that should be forwarded to Humio." +variable "humio_ingest_token" { + type = string + description = "The value of the ingest token for the repository from your Humio account to ship log/metric events to." + sensitive = true } -variable "log_retention_in_days" { - type = number - description = "The number of days to retain logs for the logs-to-humio Lambda." - default = 7 +variable "humio_lambda_log_retention" { + type = number + description = "Number of days to retain CloudWatch logs from the Humio Lambda functions." + default = 1 } -variable "memory_size" { - type = number - description = "The amount of memory for the function." - default = 128 +variable "enable_cloudwatch_logs_auto_subscription" { + type = bool + description = "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable." + default = true } -variable "private_vpn_subnet_ids" { - type = list(string) - description = "A list of subnet IDs in the private subnet of the VPN VPC." +variable "humio_cloudwatch_logs_subscription_prefix" { + type = string + description = "Humio will only subscribe to log groups with the prefix specified." + default = "" } -variable "role_permissions_boundary" { - type = string - description = "The ARN of the role permissions boundary to attach to the Lambda role." +variable "enable_cloudwatch_logs_backfiller_autorun" { + type = bool + description = "Make the backfiller run automatically when created. Set to 'true' to enable." + default = false } -variable "sub_idx_nm" { - type = string - description = "The SubIdxNM for the application in Humio." +variable "enable_vpc_for_ingester_lambdas" { + type = bool + description = "Use a VPC for the lambda ingester functions. Set to 'true' to enable." + default = false // TODO - coerce to string in parameter map } -variable "timeout" { - type = number - description = "The amount of time the function is allowed to run." - default = 30 +variable "security_group_ids" { + type = list(string) + description = "A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled." + default = [] } -variable "tags" { - type = map(string) - description = "A map of AWS Tags to attach to each resource created." - default = {} +variable "subnet_ids" { + type = list(string) + description = "A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled." + default = [] } -variable "vpn_vpc_id" { - type = string - description = "The ID of the VPC with a VPN back to campus." +variable "humio_lambda_log_level" { + type = string + description = "The log level for the Humio lambdas." + validation { + condition = contains(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], var.humio_lambda_log_level) + error_message = "Must be one of ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']" + } + default = "INFO" } - -variable "filter_pattern" { - type = string - description = "(Optional) A valid CloudWatch Logs filter pattern for subscribing to a filtered stream of log events" - default = "" -} \ No newline at end of file From 7bbb46eceac1649160a61f7f2da41fae970b7414 Mon Sep 17 00:00:00 2001 From: Brooke Date: Mon, 16 May 2022 15:07:40 -0600 Subject: [PATCH 03/42] update README inputs --- README.md | 33 +++++++++++++++++---------------- cloudformation.json | 2 +- variables.tf | 2 +- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2ac6456..33762ff 100644 --- a/README.md +++ b/README.md @@ -56,25 +56,26 @@ resources that the for_each depends on. ## Inputs -| Name | Type | Description | Default | -| --- | --- | --- | --- | -| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | -| app_name | string | The application name to include in the name of resources created. | -| humio_ingest_token | string (sensitive) | The ingest token for Humio logs. | -| log_group_arns | set(string) | The ARNs of CloudWatch log groups that should be forwarded to Humio. | -| log_group_names | set(string) | The names of the CloudWatch log groups that should be forwarded to Humio. | -| memory_size | number | The amount of memory for the function. | 128 | -| private_vpn_subnet_ids | list(string) | A list of subnet IDs in the private subnet of the VPN VPC. | -| role_permissions_boundary | string | The ARN of the role permissions boundary to attach to the Lambda role. | -| sub_idx_nm | string | The SubIdxNM for the application in Humio. | -| timeout | number | The amount of time the function is allowed to run. | 30 | -| tags | map(string) | A map of AWS Tags to attach to each resource created. | {} | -| filter_pattern | string | A valid CloudWatch Logs filter pattern for subscribing to a filtered stream of log events. | "" | +| Name | Type | Description | Default | +|-------------------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | +| app_name | string | The application name to include in the name of resources created. | | +| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | "https" | +| humio_host | string | The host to ship Humio log/metric events to. | | +| humio_ingest_token | string (sensitive) | The ingest token for Humio logs. | "" | +| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | +| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | false | +| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | +| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | +| enable_vpc_for_ingester_lambdas | bool | Use a VPC for the lambda ingester functions. Set to 'true' to enable. | false | +| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | [] | +| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | [] | +| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | ## Outputs -| Name | Type | Description | -| --- | --- | --- | +| Name | Type | Description | +|---------------|--------|-----------------------------------------| | function_name | string | The name of the logs-to-Humio function. | ## Deployment diff --git a/cloudformation.json b/cloudformation.json index b67a115..8300e49 100644 --- a/cloudformation.json +++ b/cloudformation.json @@ -44,7 +44,7 @@ "false" ], "Description" : "Make the backfiller run automatically when created. Set to 'true' to enable.", - "Default" : "true" + "Default" : "false" }, "EnableVPCForIngesterLambdas" : { "Type" : "String", diff --git a/variables.tf b/variables.tf index 280a35b..1f9d5c0 100644 --- a/variables.tf +++ b/variables.tf @@ -34,7 +34,7 @@ variable "humio_lambda_log_retention" { variable "enable_cloudwatch_logs_auto_subscription" { type = bool description = "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable." - default = true + default = false } variable "humio_cloudwatch_logs_subscription_prefix" { From ba4952056554e5da92a723bde35d604cdcd92a6d Mon Sep 17 00:00:00 2001 From: Brooke Date: Mon, 16 May 2022 15:52:35 -0600 Subject: [PATCH 04/42] fix error message & add output vars to read me --- .terraform.lock.hcl | 20 ++++++++++++++++++++ README.md | 18 +++++++++++++++--- variables.tf | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 14d856d..f47146c 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -5,6 +5,7 @@ provider "registry.terraform.io/hashicorp/aws" { version = "3.50.0" constraints = ">= 3.0.0" hashes = [ + "h1:HXowDewT9QnMg5Zpbf+VcrgPe22wbvt7jrAENc6ZENU=", "h1:gX/faGzGvZvf9W6597S7ZcMiNI1gw9ilQwM9hkc6F2Q=", "zh:11d5508e180b93ab06935dbebefb08745c78ebfe1ec41d53b4340fb7f27d32eb", "zh:192ae31ddf1c5c4ed7f64a15c8cf3f1a440b1427fa15604e90eee40037973f0a", @@ -19,3 +20,22 @@ provider "registry.terraform.io/hashicorp/aws" { "zh:eac7aad200f4df951dd30ec8547b9412909c8dc450c1322b28619499e5e2bdbe", ] } + +provider "registry.terraform.io/hashicorp/local" { + version = "2.2.2" + hashes = [ + "h1:BVEZnjtpWxKPG9OOQh4dFa1z5pwMO/uuzYtu6AR2LyM=", + "zh:027e4873c69da214e2fed131666d5de92089732a11d096b68257da54d30b6f9d", + "zh:0ba2216e16cfb72538d76a4c4945b4567a76f7edbfef926b1c5a08d7bba2a043", + "zh:1fee8f6aae1833c27caa96e156cf99a681b6f085e476d7e1b77d285e21d182c1", + "zh:2e8a3e72e877003df1c390a231e0d8e827eba9f788606e643f8e061218750360", + "zh:719008f9e262aa1523a6f9132adbe9eee93c648c2981f8359ce41a40e6425433", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9a70fdbe6ef955c4919a4519caca116f34c19c7ddedd77990fbe4f80fe66dc84", + "zh:abc412423d670cbb6264827fa80e1ffdc4a74aff3f19ba6a239dd87b85b15bec", + "zh:ae953a62c94d2a2a0822e5717fafc54e454af57bd6ed02cd301b9786765c1dd3", + "zh:be0910bdf46698560f9e86f51a4ff795c62c02f8dc82b2b1dab77a0b3a93f61e", + "zh:e58f9083b7971919b95f553227adaa7abe864fce976f0166cf4d65fc17257ff2", + "zh:ff4f77cbdbb22cc98182821c7ef84dce16298ab0e997d5c7fae97247f7a4bcb0", + ] +} diff --git a/README.md b/README.md index 33762ff..39014b1 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,21 @@ resources that the for_each depends on. ## Outputs -| Name | Type | Description | -|---------------|--------|-----------------------------------------| -| function_name | string | The name of the logs-to-Humio function. | +| Name | Type | Description | +|-------------------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | +| app_name | string | The application name to include in the name of resources created. | +| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | +| humio_host | string | The host to ship Humio log/metric events to. | +| humio_ingest_token | string (sensitive) | The ingest token for Humio logs. | +| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | +| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | +| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | +| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | +| enable_vpc_for_ingester_lambdas | bool | Use a VPC for the lambda ingester functions. Set to 'true' to enable. | +| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | +| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | +| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | ## Deployment diff --git a/variables.tf b/variables.tf index 1f9d5c0..95d2d8e 100644 --- a/variables.tf +++ b/variables.tf @@ -72,7 +72,7 @@ variable "humio_lambda_log_level" { description = "The log level for the Humio lambdas." validation { condition = contains(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], var.humio_lambda_log_level) - error_message = "Must be one of ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']" + error_message = "Must be one of ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']." } default = "INFO" } From 7542bf96126309beea56e7b2cfd3415d3f9891e4 Mon Sep 17 00:00:00 2001 From: Brooke Date: Mon, 16 May 2022 16:11:16 -0600 Subject: [PATCH 05/42] update example in README --- README.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 39014b1..04aed39 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,18 @@ to you. ```hcl module "humio_logger" { source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v1.0.0" - app_env = "dev" - app_name = "humio-logger-ci" - humio_ingest_token = "4788f2d0-b72d-484d-944d-830aba612207" // This isn't a real token - log_group_arns = [aws_cloudwatch_log_group.humio_logger.arn] - log_group_names = [aws_cloudwatch_log_group.humio_logger.name] - private_vpn_subnet_ids = module.acs_vpn.private_subnet_ids - role_permissions_boundary = module.acs_vpn.role_permissions_boundary.arn - sub_idx_nm = "payments" - vpn_vpc_id = module.acs_vpn.vpc.id + app_env = "dev" + app_name = "humio-logger-ci" + humio_host = "oit-humio.byu.edu" // module.acs_vpn.private_subnet_ids + humio_ingest_token = module.acs.humio.prd.token + humio_lambda_log_retention = 7 + enable_cloudwatch_logs_auto_subscription = false + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" + enable_cloudwatch_logs_backfiller_autorun = false + enable_vpc_for_ingester_lambdas = true + security_group_ids = module.acs.vpc.id // TODO: better example + subnet_ids = module.acs.asd // TODO: better example + humio_lambda_log_level = "ERROR" } ``` From 553f8300213d2ba1b4bf2004e290f5200d09370f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 22:45:56 +0000 Subject: [PATCH 06/42] Bump actions/checkout from 2 to 3 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6c37b2..83f2dae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: name: Terraform Format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Terraform Setup uses: hashicorp/setup-terraform@v1 @@ -31,7 +31,7 @@ jobs: matrix: terraform: ['0.14.x', '0.15.x', '1.0.x'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 From c30140c8abd9b3f0e9a4b47eb9308fd2ec840f7a Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Tue, 17 May 2022 11:56:08 -0600 Subject: [PATCH 07/42] simplify and cleanup --- README.md | 50 ++++++-------------- main.tf | 131 +++++++++++---------------------------------------- variables.tf | 55 ++++----------------- 3 files changed, 51 insertions(+), 185 deletions(-) diff --git a/README.md b/README.md index 04aed39..79a12a3 100644 --- a/README.md +++ b/README.md @@ -10,51 +10,27 @@ Infrastructure to watch a CloudWatch log group and forward its logs to Humio. ### Prerequisites -Before including this module in your project, be sure to communicate with the Platform Engineer over Humio (currently -Carson Mills) so that he can keep an updated list of the various data streams coming into Humio and who is responsible -for them. He will also need to update filters on views to include the new SubIdxNM values so that data will be visible +Before including this module in your project, be sure to communicate with the Platform Engineer over Humio (currently +Carson Mills) so that he can keep an updated list of the various data streams coming into Humio and who is responsible +for them. He will also need to update filters on views to include the new SubIdxNM values so that data will be visible to you. ### Example ```hcl module "humio_logger" { - source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v1.0.0" - app_env = "dev" - app_name = "humio-logger-ci" - humio_host = "oit-humio.byu.edu" // module.acs_vpn.private_subnet_ids - humio_ingest_token = module.acs.humio.prd.token - humio_lambda_log_retention = 7 - enable_cloudwatch_logs_auto_subscription = false - humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" - enable_cloudwatch_logs_backfiller_autorun = false - enable_vpc_for_ingester_lambdas = true - security_group_ids = module.acs.vpc.id // TODO: better example - subnet_ids = module.acs.asd // TODO: better example - humio_lambda_log_level = "ERROR" + source = "github.com/byu-oit/terraform-aws-humio-logger?ref=0.0.0-humio" + app_env = "dev" + app_name = "humio-logger-ci" + humio_host = module.acs.humio_prd_endpoint + humio_ingest_token = module.acs.humio_prd_token + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" } ``` -### Limitations - -Because of [limitations with Terraform](https://www.terraform.io/docs/language/meta-arguments/for_each.html#limitations-on-values-used-in-for_each), -add this module after the initial deployment of your application to a new environment. Otherwise, you'll get an error -similar to this: - -```bash -Error: Invalid for_each argument - on ../../main.tf line 75, in resource "aws_lambda_permission" "logging": - 75: for_each = var.log_group_arns - -The "for_each" value depends on resource attributes that cannot be determined -until apply, so Terraform cannot predict how many instances will be created. -To work around this, use the -target argument to first apply only the -resources that the for_each depends on. -``` - ## Requirements -* Terraform version 0.14.11 or greater +* Terraform version 0.12.17 or greater * AWS provider version 3.0 or greater ## Inputs @@ -100,6 +76,8 @@ If you update the Lambda function code, be sure to run `zip -r function.zip .` i ## Development To update to the latest bundle of Cloudwatch2Humio: + 1. Clone the git repository: [https://github.com/humio/cloudwatch2humio](https://github.com/humio/cloudwatch2humio) -4. Copy the `$PROJECT/cloudformation.json` file to this project replacing the old cloudformation template. -5. Ensure that the necessary refactors are implemented in terraform. +2. Copy the `$PROJECT/cloudformation.json` file to this project replacing the old cloudformation template. This file + should not be modified. +3. Ensure that the necessary refactors are implemented in terraform. diff --git a/main.tf b/main.tf index 7b75c8f..1e7136a 100644 --- a/main.tf +++ b/main.tf @@ -1,123 +1,46 @@ terraform { - required_version = ">= 0.14.11" + required_version = ">= 0.12.17" required_providers { aws = ">= 3.0" } } module "acs" { - source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.4.1" + source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.5.0" + vpc_vpn_to_campus = true } data "local_file" "cloudformation" { filename = "${path.module}/cloudformation.json" } -resource "aws_cloudformation_stack" "cloudwatch" { - name = "${var.app_name}-cloudwatch2humio" +resource "aws_security_group" "logging" { + name = "${var.app_name}-logs-to-humio" + vpc_id = module.acs.vpc.id + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } +} + +resource "aws_cloudformation_stack" "cloudwatch" { + name = "${var.app_name}-cloudwatch2humio" + template_body = data.local_file.cloudformation.content parameters = { - HumioProtocol = var.humio_protocol - HumioHost = var.humio_host - HumioIngestToken = var.humio_ingest_token - HumioLambdaLogRetention = var.humio_lambda_log_retention - EnableCloudWatchLogsAutoSubscription = tostring(var.enable_cloudwatch_logs_auto_subscription) + HumioProtocol = "https" + HumioHost = var.app_env == "prd" ? module.acs.humio_prd_endpoint : module.acs.humio_dev_endpoint + HumioIngestToken = var.app_env == "prd" ? module.acs.humio_prd_token : module.acs.humio_dev_token + HumioLambdaLogRetention = var.humio_lambda_log_retention + EnableCloudWatchLogsAutoSubscription = tostring(var.enable_cloudwatch_logs_auto_subscription) HumioCloudWatchLogsSubscriptionPrefix = var.humio_cloudwatch_logs_subscription_prefix EnableCloudWatchLogsBackfillerAutoRun = tostring(var.enable_cloudwatch_logs_backfiller_autorun) - EnableVPCForIngesterLambdas = tostring(var.enable_vpc_for_ingester_lambdas) - SecurityGroupIds = var.security_group_ids - SubnetIds = var.subnet_ids - HumioLambdaLogLevel = var.humio_lambda_log_level + EnableVPCForIngesterLambdas = "true" + SecurityGroupIds = [aws_security_group.logging.id] + SubnetIds = module.acs.private_subnet_ids + HumioLambdaLogLevel = var.humio_lambda_log_level } - - template_body = data.local_file.cloudformation.content } - -#resource "aws_lambda_function" "logging" { -# function_name = "${var.app_name}-logs-to-humio" -# filename = local.filename -# handler = "index.handler" -# runtime = "nodejs14.x" -# role = aws_iam_role.logging.arn -# source_code_hash = filebase64sha256(local.filename) -# timeout = var.timeout -# memory_size = var.memory_size -# tags = var.tags -# -# environment { -# variables = { -# ENV = var.app_env -# HUMIO_INGEST_TOKEN = var.humio_ingest_token -# SUB_IDX_NM = var.sub_idx_nm -# } -# } -# -# vpc_config { -# security_group_ids = [aws_security_group.logging.id] -# subnet_ids = var.private_vpn_subnet_ids -# } -#} -# -#resource "aws_security_group" "logging" { -# name = "${var.app_name}-logs-to-humio" -# vpc_id = var.vpn_vpc_id -# -# egress { -# from_port = 0 -# to_port = 0 -# protocol = "-1" -# cidr_blocks = ["0.0.0.0/0"] -# ipv6_cidr_blocks = ["::/0"] -# } -#} -# -#resource "aws_iam_role" "logging" { -# name = "${var.app_name}-logs-to-humio-role" -# permissions_boundary = var.role_permissions_boundary -# assume_role_policy = jsonencode({ -# Version = "2012-10-17", -# Statement = [ -# { -# Action = "sts:AssumeRole", -# Principal = { -# Service = "lambda.amazonaws.com" -# }, -# Effect = "Allow", -# Sid = "" -# } -# ] -# }) -# tags = var.tags -#} -# -#resource "aws_iam_role_policy_attachment" "basic_execution" { -# policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" -# role = aws_iam_role.logging.name -#} -# -#resource "aws_iam_role_policy_attachment" "logging_eni_attach" { -# policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaENIManagementAccess" -# role = aws_iam_role.logging.name -#} -# -#resource "aws_lambda_permission" "logging" { -# for_each = var.log_group_arns -# action = "lambda:InvokeFunction" -# function_name = aws_lambda_function.logging.function_name -# principal = "logs.amazonaws.com" -# source_arn = "${each.key}:*" -#} -# -#resource "aws_cloudwatch_log_subscription_filter" "logging" { -# for_each = var.log_group_names -# depends_on = [aws_lambda_permission.logging] -# destination_arn = aws_lambda_function.logging.arn -# filter_pattern = var.filter_pattern -# log_group_name = each.key -# name = "logging_default" -#} -# -#resource "aws_cloudwatch_log_group" "logging" { -# name = "/aws/lambda/${aws_lambda_function.logging.function_name}" -# retention_in_days = var.log_retention_in_days -#} diff --git a/variables.tf b/variables.tf index 95d2d8e..91ad4cc 100644 --- a/variables.tf +++ b/variables.tf @@ -8,70 +8,35 @@ variable "app_name" { description = "The application name to include in the name of resources created." } -variable "humio_protocol" { - type = string - description = "The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended." - default = "https" -} - -variable "humio_host" { - type = string - description = "The host to ship Humio log/metric events to." -} - -variable "humio_ingest_token" { - type = string - description = "The value of the ingest token for the repository from your Humio account to ship log/metric events to." - sensitive = true -} - variable "humio_lambda_log_retention" { - type = number + type = number description = "Number of days to retain CloudWatch logs from the Humio Lambda functions." - default = 1 + default = 1 } variable "enable_cloudwatch_logs_auto_subscription" { - type = bool + type = bool description = "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable." - default = false + default = false } variable "humio_cloudwatch_logs_subscription_prefix" { - type = string + type = string description = "Humio will only subscribe to log groups with the prefix specified." - default = "" + default = "" // Defaults to all log groups } variable "enable_cloudwatch_logs_backfiller_autorun" { - type = bool + type = bool description = "Make the backfiller run automatically when created. Set to 'true' to enable." - default = false -} - -variable "enable_vpc_for_ingester_lambdas" { - type = bool - description = "Use a VPC for the lambda ingester functions. Set to 'true' to enable." - default = false // TODO - coerce to string in parameter map -} - -variable "security_group_ids" { - type = list(string) - description = "A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled." - default = [] -} - -variable "subnet_ids" { - type = list(string) - description = "A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled." - default = [] + default = false } variable "humio_lambda_log_level" { - type = string + type = string description = "The log level for the Humio lambdas." validation { - condition = contains(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], var.humio_lambda_log_level) + condition = contains(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], var.humio_lambda_log_level) error_message = "Must be one of ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']." } default = "INFO" From a2d883c7971ee4c5dc00f0db80cdc0fb3fcfd679 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Tue, 17 May 2022 13:02:47 -0600 Subject: [PATCH 08/42] ci: fix ci and examples --- README.md | 2 -- examples/ci/ci.tf | 17 +---------------- examples/simple/simple-example.tf | 25 ++++--------------------- 3 files changed, 5 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 79a12a3..1354a24 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,6 @@ module "humio_logger" { source = "github.com/byu-oit/terraform-aws-humio-logger?ref=0.0.0-humio" app_env = "dev" app_name = "humio-logger-ci" - humio_host = module.acs.humio_prd_endpoint - humio_ingest_token = module.acs.humio_prd_token humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" } ``` diff --git a/examples/ci/ci.tf b/examples/ci/ci.tf index e3aaf95..a8b6769 100644 --- a/examples/ci/ci.tf +++ b/examples/ci/ci.tf @@ -17,20 +17,5 @@ module "ci_test" { source = "../../" app_env = "dev" app_name = "humio-logger-ci" - humio_ingest_token = "4788f2d0-b72d-484d-944d-830aba612207" - log_group_arns = [data.aws_cloudwatch_log_group.humio_logger.arn] - log_group_names = [data.aws_cloudwatch_log_group.humio_logger.name] - private_vpn_subnet_ids = module.acs_vpn.private_subnet_ids - role_permissions_boundary = module.acs_vpn.role_permissions_boundary.arn - sub_idx_nm = "terraform" - vpn_vpc_id = module.acs_vpn.vpc.id + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" } - -data "aws_cloudwatch_log_group" "humio_logger" { - name = "humio-logger-ci-logs" -} - -module "acs_vpn" { - source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.2.0" - vpc_vpn_to_campus = true -} \ No newline at end of file diff --git a/examples/simple/simple-example.tf b/examples/simple/simple-example.tf index 1d62dc7..6b87aed 100644 --- a/examples/simple/simple-example.tf +++ b/examples/simple/simple-example.tf @@ -10,25 +10,8 @@ provider "aws" { } module "humio_logger" { - source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v1.0.0" - // source = "../" # for local testing during module development - app_env = "dev" - app_name = "humio-logger-ci" - humio_ingest_token = "4788f2d0-b72d-484d-944d-830aba612207" - log_group_arns = [aws_cloudwatch_log_group.humio_logger.arn] - log_group_names = [aws_cloudwatch_log_group.humio_logger.name] - private_vpn_subnet_ids = module.acs_vpn.private_subnet_ids - role_permissions_boundary = module.acs_vpn.role_permissions_boundary.arn - sub_idx_nm = "payments" - vpn_vpc_id = module.acs_vpn.vpc.id -} - -resource "aws_cloudwatch_log_group" "humio_logger" { - name = "humio-logger-ci-logs" - retention_in_days = 7 -} - -module "acs_vpn" { - source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.2.0" - vpc_vpn_to_campus = true + source = "github.com/byu-oit/terraform-aws-humio-logger?ref=0.0.0-humio" + app_env = "dev" + app_name = "humio-logger-ci" + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" } From dd19afa27955a10a276b9f970fbc82d21290bd7a Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Tue, 17 May 2022 13:12:28 -0600 Subject: [PATCH 09/42] docs: update outputs --- README.md | 12 ------------ examples/ci/ci.tf | 10 +++++----- examples/simple/simple-example.tf | 2 +- outputs.tf | 24 ------------------------ 4 files changed, 6 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 1354a24..fc2e53a 100644 --- a/README.md +++ b/README.md @@ -37,16 +37,10 @@ module "humio_logger" { |-------------------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------|---------| | app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | | app_name | string | The application name to include in the name of resources created. | | -| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | "https" | -| humio_host | string | The host to ship Humio log/metric events to. | | -| humio_ingest_token | string (sensitive) | The ingest token for Humio logs. | "" | | humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | | enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | false | | humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | | enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | -| enable_vpc_for_ingester_lambdas | bool | Use a VPC for the lambda ingester functions. Set to 'true' to enable. | false | -| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | [] | -| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | [] | | humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | ## Outputs @@ -55,16 +49,10 @@ module "humio_logger" { |-------------------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| | app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | app_name | string | The application name to include in the name of resources created. | -| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | -| humio_host | string | The host to ship Humio log/metric events to. | -| humio_ingest_token | string (sensitive) | The ingest token for Humio logs. | | humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | | enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | | humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | | enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | -| enable_vpc_for_ingester_lambdas | bool | Use a VPC for the lambda ingester functions. Set to 'true' to enable. | -| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | -| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | | humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | ## Deployment diff --git a/examples/ci/ci.tf b/examples/ci/ci.tf index a8b6769..6771dec 100644 --- a/examples/ci/ci.tf +++ b/examples/ci/ci.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.14.11" + required_version = ">= 0.12.17" required_providers { aws = { @@ -14,8 +14,8 @@ provider "aws" { } module "ci_test" { - source = "../../" - app_env = "dev" - app_name = "humio-logger-ci" - humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" + source = "../../" + app_env = "dev" + app_name = "humio-logger-ci" + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" } diff --git a/examples/simple/simple-example.tf b/examples/simple/simple-example.tf index 6b87aed..8e255a6 100644 --- a/examples/simple/simple-example.tf +++ b/examples/simple/simple-example.tf @@ -13,5 +13,5 @@ module "humio_logger" { source = "github.com/byu-oit/terraform-aws-humio-logger?ref=0.0.0-humio" app_env = "dev" app_name = "humio-logger-ci" - humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" } diff --git a/outputs.tf b/outputs.tf index 4e5a0cf..eaaa870 100644 --- a/outputs.tf +++ b/outputs.tf @@ -6,18 +6,6 @@ output "app_name" { value = var.app_name } -output "humio_protocol" { - value = var.humio_protocol -} - -output "humio_host" { - value = var.humio_host -} - -output "humio_ingest_token" { - value = var.humio_ingest_token -} - output "humio_lambda_log_retention" { value = var.humio_lambda_log_retention } @@ -34,18 +22,6 @@ output "enable_cloudwatch_logs_backfiller_autorun" { value = var.enable_cloudwatch_logs_backfiller_autorun } -output "enable_vpc_for_ingester_lambdas" { - value = var.enable_vpc_for_ingester_lambdas -} - -output "security_group_ids" { - value = var.security_group_ids -} - -output "subnet_ids" { - value = var.subnet_ids -} - output "humio_lambda_log_level" { value = var.humio_lambda_log_level } From 5f805ac8956e36ab33b4407c5401d1f3fdd6ada9 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Tue, 17 May 2022 13:19:22 -0600 Subject: [PATCH 10/42] fix: correct typo --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 1e7136a..0d501e3 100644 --- a/main.tf +++ b/main.tf @@ -27,7 +27,7 @@ resource "aws_security_group" "logging" { } } -resource "aws_cloudformation_stack" "cloudwatch" { +resource "aws_cloudformation_stack" "cloudwatch2humio" { name = "${var.app_name}-cloudwatch2humio" template_body = data.local_file.cloudformation.content parameters = { @@ -39,7 +39,7 @@ resource "aws_cloudformation_stack" "cloudwatch" { HumioCloudWatchLogsSubscriptionPrefix = var.humio_cloudwatch_logs_subscription_prefix EnableCloudWatchLogsBackfillerAutoRun = tostring(var.enable_cloudwatch_logs_backfiller_autorun) EnableVPCForIngesterLambdas = "true" - SecurityGroupIds = [aws_security_group.logging.id] + SecurityGroups = [aws_security_group.logging.id] SubnetIds = module.acs.private_subnet_ids HumioLambdaLogLevel = var.humio_lambda_log_level } From 52d22be1ca13c5da898ccdd7642026f74b97c578 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Tue, 17 May 2022 13:34:38 -0600 Subject: [PATCH 11/42] feat: add version variable as input --- README.md | 24 +++++++++++++----------- main.tf | 5 +++-- outputs.tf | 4 ++++ variables.tf | 11 +++++++++-- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index fc2e53a..209d90b 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ module "humio_logger" { source = "github.com/byu-oit/terraform-aws-humio-logger?ref=0.0.0-humio" app_env = "dev" app_name = "humio-logger-ci" - humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" } ``` @@ -42,18 +42,20 @@ module "humio_logger" { | humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | | enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | | humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | +| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | ## Outputs -| Name | Type | Description | -|-------------------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| -| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | -| app_name | string | The application name to include in the name of resources created. | -| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | -| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | -| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | -| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | -| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | +| Name | Type | Description | +|-------------------------------------------|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | +| app_name | string | The application name to include in the name of resources created. | +| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | +| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | +| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | +| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | +| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | +| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | ## Deployment @@ -61,7 +63,7 @@ If you update the Lambda function code, be sure to run `zip -r function.zip .` i ## Development -To update to the latest bundle of Cloudwatch2Humio: +To update to the latest version of Cloudwatch2Humio: 1. Clone the git repository: [https://github.com/humio/cloudwatch2humio](https://github.com/humio/cloudwatch2humio) 2. Copy the `$PROJECT/cloudformation.json` file to this project replacing the old cloudformation template. This file diff --git a/main.tf b/main.tf index 0d501e3..248c3c4 100644 --- a/main.tf +++ b/main.tf @@ -39,8 +39,9 @@ resource "aws_cloudformation_stack" "cloudwatch2humio" { HumioCloudWatchLogsSubscriptionPrefix = var.humio_cloudwatch_logs_subscription_prefix EnableCloudWatchLogsBackfillerAutoRun = tostring(var.enable_cloudwatch_logs_backfiller_autorun) EnableVPCForIngesterLambdas = "true" - SecurityGroups = [aws_security_group.logging.id] - SubnetIds = module.acs.private_subnet_ids + SecurityGroupIds = join(", ", [aws_security_group.logging.id]) + SubnetIds = join(", ", module.acs.private_subnet_ids) HumioLambdaLogLevel = var.humio_lambda_log_level + Version = var.cloudwatch2humio_version } } diff --git a/outputs.tf b/outputs.tf index eaaa870..2aa0d1e 100644 --- a/outputs.tf +++ b/outputs.tf @@ -25,3 +25,7 @@ output "enable_cloudwatch_logs_backfiller_autorun" { output "humio_lambda_log_level" { value = var.humio_lambda_log_level } + +output "cloudwatch2humio_version" { + value = var.cloudwatch2humio_version +} diff --git a/variables.tf b/variables.tf index 91ad4cc..9851162 100644 --- a/variables.tf +++ b/variables.tf @@ -16,8 +16,8 @@ variable "humio_lambda_log_retention" { variable "enable_cloudwatch_logs_auto_subscription" { type = bool - description = "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable." - default = false + description = "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'false' to disable." + default = true } variable "humio_cloudwatch_logs_subscription_prefix" { @@ -41,3 +41,10 @@ variable "humio_lambda_log_level" { } default = "INFO" } + +variable "cloudwatch2humio_version" { + type = string + description = "The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository." + nullable = true + default = null +} From 50ce7521d33b4ca5864755394e719e0825668ac7 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Tue, 17 May 2022 13:49:36 -0600 Subject: [PATCH 12/42] fix: requires v0.14 of terraform --- README.md | 2 +- examples/ci/ci.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 209d90b..d364467 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ module "humio_logger" { ## Requirements -* Terraform version 0.12.17 or greater +* Terraform version 0.14 or greater * AWS provider version 3.0 or greater ## Inputs diff --git a/examples/ci/ci.tf b/examples/ci/ci.tf index 6771dec..45cdcd8 100644 --- a/examples/ci/ci.tf +++ b/examples/ci/ci.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.12.17" + required_version = ">= 0.14" required_providers { aws = { From ced46635afd9f62bc64e99f045f41d998812d87c Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Tue, 17 May 2022 13:52:10 -0600 Subject: [PATCH 13/42] fix: correct default --- variables.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/variables.tf b/variables.tf index 9851162..cb700cf 100644 --- a/variables.tf +++ b/variables.tf @@ -45,6 +45,5 @@ variable "humio_lambda_log_level" { variable "cloudwatch2humio_version" { type = string description = "The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository." - nullable = true - default = null + default = "v1.2.1" } From 1f675e5912fb4dfae5fd0f8fe47c419083084288 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Tue, 17 May 2022 14:11:00 -0600 Subject: [PATCH 14/42] docs: update version --- README.md | 2 +- examples/simple/simple-example.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d364467..61d35ae 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ to you. ```hcl module "humio_logger" { - source = "github.com/byu-oit/terraform-aws-humio-logger?ref=0.0.0-humio" + source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" app_env = "dev" app_name = "humio-logger-ci" humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" diff --git a/examples/simple/simple-example.tf b/examples/simple/simple-example.tf index 8e255a6..6e1e2c9 100644 --- a/examples/simple/simple-example.tf +++ b/examples/simple/simple-example.tf @@ -10,7 +10,7 @@ provider "aws" { } module "humio_logger" { - source = "github.com/byu-oit/terraform-aws-humio-logger?ref=0.0.0-humio" + source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v1.0.0-humio" app_env = "dev" app_name = "humio-logger-ci" humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" From 66b779b155fa45c1c0a408dca5035e4a01db40a5 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Tue, 17 May 2022 14:12:23 -0600 Subject: [PATCH 15/42] ci: remove code analysis (no longer needed) --- .github/workflows/codeql-analysis.yml | 72 --------------------------- 1 file changed, 72 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index e759846..0000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,72 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ main ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ main ] - schedule: - - cron: '31 17 * * 1' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'javascript' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 From 91de9059ff988f73c8485207afafb03f4223eb45 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Tue, 17 May 2022 14:15:59 -0600 Subject: [PATCH 16/42] docs: fix version --- examples/simple/simple-example.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple/simple-example.tf b/examples/simple/simple-example.tf index 6e1e2c9..e6314d4 100644 --- a/examples/simple/simple-example.tf +++ b/examples/simple/simple-example.tf @@ -10,7 +10,7 @@ provider "aws" { } module "humio_logger" { - source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v1.0.0-humio" + source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" app_env = "dev" app_name = "humio-logger-ci" humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" From 4465a5cb11ddb45ef2906cb64520017f0f72f828 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Wed, 18 May 2022 11:52:12 -0600 Subject: [PATCH 17/42] fix: remove acs dependency --- README.md | 52 +++++++++++++++++++------------ cloudformation.json | 2 +- examples/ci/ci.tf | 13 ++++++-- examples/simple/simple-example.tf | 19 +++++++++-- main.tf | 27 ++++++++-------- outputs.tf | 24 ++++++++++++++ variables.tf | 49 +++++++++++++++++++++++++++++ 7 files changed, 147 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 61d35ae..1296850 100644 --- a/README.md +++ b/README.md @@ -33,29 +33,41 @@ module "humio_logger" { ## Inputs -| Name | Type | Description | Default | -|-------------------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------|---------| -| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | -| app_name | string | The application name to include in the name of resources created. | | -| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | -| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | false | -| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | -| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | -| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | -| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | +| Name | Type | Description | Default | +|-------------------------------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | +| app_name | string | The application name to include in the name of resources created. | | +| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | | +| humio_host | string | The host to ship Humio log/metric events to. | | +| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | | +| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | +| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | false | +| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | +| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | +| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | | +| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | | +| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | | +| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | +| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | ## Outputs -| Name | Type | Description | -|-------------------------------------------|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | -| app_name | string | The application name to include in the name of resources created. | -| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | -| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | -| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | -| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | -| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | -| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | +| Name | Type | Description | Default | +|-------------------------------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | +| app_name | string | The application name to include in the name of resources created. | | +| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | | +| humio_host | string | The host to ship Humio log/metric events to. | | +| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | | +| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | +| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | false | +| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | +| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | +| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | | +| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | | +| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | | +| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | +| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | ## Deployment diff --git a/cloudformation.json b/cloudformation.json index 8300e49..dca9018 100644 --- a/cloudformation.json +++ b/cloudformation.json @@ -61,7 +61,7 @@ }, "SubnetIds" : { "Type" : "CommaDelimitedList", - "Description" : "A comma separated list of subnet ids used by the VPC configuration that the ingester lamda functions will be deployed into. Only required if VPC is enabled." + "Description" : "A comma separated list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled." }, "HumioLambdaLogLevel" : { "Type" : "String", diff --git a/examples/ci/ci.tf b/examples/ci/ci.tf index 45cdcd8..ececa6e 100644 --- a/examples/ci/ci.tf +++ b/examples/ci/ci.tf @@ -13,9 +13,18 @@ provider "aws" { region = "us-west-2" } -module "ci_test" { - source = "../../" +module "acs" { + source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.5.0" + vpc_vpn_to_campus = true +} + +module "humio_logger" { + source = "../.." app_env = "dev" app_name = "humio-logger-ci" humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" + vpc_id = module.acs.vpc.id + subnet_ids = module.acs.private_subnet_ids + humio_host = module.acs.humio_dev_endpoint + humio_ingest_token = module.acs.humio_dev_token } diff --git a/examples/simple/simple-example.tf b/examples/simple/simple-example.tf index e6314d4..88b1a01 100644 --- a/examples/simple/simple-example.tf +++ b/examples/simple/simple-example.tf @@ -1,7 +1,11 @@ terraform { - aws = { - source = "hashicorp/aws" - version = "~> 3.0" + required_version = ">= 0.14" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } } } @@ -9,9 +13,18 @@ provider "aws" { region = "us-west-2" } +module "acs" { + source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.5.0" + vpc_vpn_to_campus = true +} + module "humio_logger" { source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" app_env = "dev" app_name = "humio-logger-ci" humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" + vpc_id = module.acs.vpc.id + subnet_ids = module.acs.private_subnet_ids + humio_host = module.acs.humio_dev_endpoint + humio_ingest_token = module.acs.humio_dev_token } diff --git a/main.tf b/main.tf index 248c3c4..bf07867 100644 --- a/main.tf +++ b/main.tf @@ -1,22 +1,23 @@ terraform { - required_version = ">= 0.12.17" + required_version = ">= 0.14" required_providers { aws = ">= 3.0" } } -module "acs" { - source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.5.0" - vpc_vpn_to_campus = true -} - data "local_file" "cloudformation" { filename = "${path.module}/cloudformation.json" } +locals { + enable_vpc_for_ingester_lambdas = length(var.vpc_id) > 0 ? true : false + security_group_ids = length(var.security_group_ids) > 0 ? var.security_group_ids : [aws_security_group.logging.id] +} + resource "aws_security_group" "logging" { + count = local.enable_vpc_for_ingester_lambdas && length(local.security_group_ids) <= 0 ? 1 : 0 name = "${var.app_name}-logs-to-humio" - vpc_id = module.acs.vpc.id + vpc_id = var.vpc_id egress { from_port = 0 @@ -31,16 +32,16 @@ resource "aws_cloudformation_stack" "cloudwatch2humio" { name = "${var.app_name}-cloudwatch2humio" template_body = data.local_file.cloudformation.content parameters = { - HumioProtocol = "https" - HumioHost = var.app_env == "prd" ? module.acs.humio_prd_endpoint : module.acs.humio_dev_endpoint - HumioIngestToken = var.app_env == "prd" ? module.acs.humio_prd_token : module.acs.humio_dev_token + HumioProtocol = var.humio_protocol + HumioHost = var.humio_host + HumioIngestToken = var.humio_ingest_token HumioLambdaLogRetention = var.humio_lambda_log_retention EnableCloudWatchLogsAutoSubscription = tostring(var.enable_cloudwatch_logs_auto_subscription) HumioCloudWatchLogsSubscriptionPrefix = var.humio_cloudwatch_logs_subscription_prefix EnableCloudWatchLogsBackfillerAutoRun = tostring(var.enable_cloudwatch_logs_backfiller_autorun) - EnableVPCForIngesterLambdas = "true" - SecurityGroupIds = join(", ", [aws_security_group.logging.id]) - SubnetIds = join(", ", module.acs.private_subnet_ids) + EnableVPCForIngesterLambdas = tostring(local.enable_vpc_for_ingester_lambdas) + SecurityGroupIds = join(", ", local.security_group_ids) + SubnetIds = join(", ", var.subnet_ids) HumioLambdaLogLevel = var.humio_lambda_log_level Version = var.cloudwatch2humio_version } diff --git a/outputs.tf b/outputs.tf index 2aa0d1e..cc60587 100644 --- a/outputs.tf +++ b/outputs.tf @@ -6,6 +6,18 @@ output "app_name" { value = var.app_name } +output "humio_protocol" { + value = var.humio_protocol +} + +output "humio_host" { + value = var.humio_host +} + +output "humio_ingest_token" { + value = var.humio_ingest_token +} + output "humio_lambda_log_retention" { value = var.humio_lambda_log_retention } @@ -22,6 +34,18 @@ output "enable_cloudwatch_logs_backfiller_autorun" { value = var.enable_cloudwatch_logs_backfiller_autorun } +output "vpc_id" { + value = var.vpc_id +} + +output "security_group_id" { + value = local.security_group_ids +} + +output "subnet_ids" { + value = var.subnet_ids +} + output "humio_lambda_log_level" { value = var.humio_lambda_log_level } diff --git a/variables.tf b/variables.tf index cb700cf..cea7cfd 100644 --- a/variables.tf +++ b/variables.tf @@ -8,6 +8,29 @@ variable "app_name" { description = "The application name to include in the name of resources created." } +variable "humio_protocol" { + type = string + description = "The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended." + validation { + condition = contains(["HTTPS", "HTTP"], var.humio_protocol) + error_message = "Must be one of ['HTTPS', 'HTTP']." + } + default = "HTTPS" +} + +variable "humio_host" { + type = string + description = "The host to ship Humio log/metric events to." + default = "cloud.humio.com" +} + +variable "humio_ingest_token" { + type = string + description = "The value of the ingest token for the repository from your Humio account to ship log/metric events to." + default = "" + sensitive = true +} + variable "humio_lambda_log_retention" { type = number description = "Number of days to retain CloudWatch logs from the Humio Lambda functions." @@ -32,6 +55,32 @@ variable "enable_cloudwatch_logs_backfiller_autorun" { default = false } +variable "vpc_id" { + type = string + description = "Use a VPC for the lambda ingester functions. Pass in a vpc to enable." + default = "" +} + +variable "security_group_ids" { + type = list(string) + description = "A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled." + validation { + condition = local.enable_vpc_for_ingester_lambdas + error_message = "Only required if VPC is enabled." + } + default = [] +} + +variable "subnet_ids" { + type = list(string) + description = "A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled." + validation { + condition = local.enable_vpc_for_ingester_lambdas && length(var.subnet_ids) <= 0 + error_message = "Only required if VPC is enabled." + } + default = [] +} + variable "humio_lambda_log_level" { type = string description = "The log level for the Humio lambdas." From 7097337c2491e662f5f06bab4822672c5ba54aa7 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Wed, 18 May 2022 11:58:58 -0600 Subject: [PATCH 18/42] ci: add terraform version to test --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6c37b2..77242df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - terraform: ['0.14.x', '0.15.x', '1.0.x'] + terraform: ['0.14.x', '0.15.x', '1.0.x', '1.1.0'] steps: - uses: actions/checkout@v2 From f32fe338d4f98e8634713321dd8a8676977c9a62 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Wed, 18 May 2022 12:01:40 -0600 Subject: [PATCH 19/42] fix: remove invalid conditional --- main.tf | 3 ++- variables.tf | 12 ++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/main.tf b/main.tf index bf07867..faeb373 100644 --- a/main.tf +++ b/main.tf @@ -11,7 +11,8 @@ data "local_file" "cloudformation" { locals { enable_vpc_for_ingester_lambdas = length(var.vpc_id) > 0 ? true : false - security_group_ids = length(var.security_group_ids) > 0 ? var.security_group_ids : [aws_security_group.logging.id] + subnet_ids = local.enable_vpc_for_ingester_lambdas ? var.subnet_ids : [] + security_group_ids = local.enable_vpc_for_ingester_lambdas ? length(var.security_group_ids) > 0 ? var.security_group_ids : [aws_security_group.logging.id] : [] } resource "aws_security_group" "logging" { diff --git a/variables.tf b/variables.tf index cea7cfd..d0a96f0 100644 --- a/variables.tf +++ b/variables.tf @@ -64,21 +64,13 @@ variable "vpc_id" { variable "security_group_ids" { type = list(string) description = "A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled." - validation { - condition = local.enable_vpc_for_ingester_lambdas - error_message = "Only required if VPC is enabled." - } - default = [] + default = [] } variable "subnet_ids" { type = list(string) description = "A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled." - validation { - condition = local.enable_vpc_for_ingester_lambdas && length(var.subnet_ids) <= 0 - error_message = "Only required if VPC is enabled." - } - default = [] + default = [] } variable "humio_lambda_log_level" { From 489285ce5d9126d454b6db45739e4ea20b055b08 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Wed, 18 May 2022 12:14:02 -0600 Subject: [PATCH 20/42] ci: do not fail fast on plan --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77242df..522769a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: - name: Terraform Setup uses: hashicorp/setup-terraform@v1 with: - terraform_version: '0.15.x' + terraform_version: '1.1.x' - name: Terraform Format working-directory: "./" @@ -29,7 +29,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - terraform: ['0.14.x', '0.15.x', '1.0.x', '1.1.0'] + terraform: ['0.14.x', '0.15.x', '1.0.x', '1.1.x'] + fail-fast: false steps: - uses: actions/checkout@v2 From 222ecada076ace657a1391fd48c1aa31254dfb43 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Wed, 18 May 2022 13:10:07 -0600 Subject: [PATCH 21/42] fix: remove cyclical dependency --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index faeb373..2181e11 100644 --- a/main.tf +++ b/main.tf @@ -16,7 +16,7 @@ locals { } resource "aws_security_group" "logging" { - count = local.enable_vpc_for_ingester_lambdas && length(local.security_group_ids) <= 0 ? 1 : 0 + count = local.enable_vpc_for_ingester_lambdas && length(var.security_group_ids) <= 0 ? 1 : 0 name = "${var.app_name}-logs-to-humio" vpc_id = var.vpc_id From db11e36b52e1fadab73254f3346d0c80bea9ca0d Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Wed, 18 May 2022 13:17:28 -0600 Subject: [PATCH 22/42] fix: remove sensitive output, update resource reference --- main.tf | 2 +- outputs.tf | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/main.tf b/main.tf index 2181e11..2a32e1c 100644 --- a/main.tf +++ b/main.tf @@ -12,7 +12,7 @@ data "local_file" "cloudformation" { locals { enable_vpc_for_ingester_lambdas = length(var.vpc_id) > 0 ? true : false subnet_ids = local.enable_vpc_for_ingester_lambdas ? var.subnet_ids : [] - security_group_ids = local.enable_vpc_for_ingester_lambdas ? length(var.security_group_ids) > 0 ? var.security_group_ids : [aws_security_group.logging.id] : [] + security_group_ids = local.enable_vpc_for_ingester_lambdas ? length(var.security_group_ids) > 0 ? var.security_group_ids : [aws_security_group.logging[0].id] : [] } resource "aws_security_group" "logging" { diff --git a/outputs.tf b/outputs.tf index cc60587..3381e5e 100644 --- a/outputs.tf +++ b/outputs.tf @@ -14,10 +14,6 @@ output "humio_host" { value = var.humio_host } -output "humio_ingest_token" { - value = var.humio_ingest_token -} - output "humio_lambda_log_retention" { value = var.humio_lambda_log_retention } From 97450409d9a62800cd8e30cdecc2acc937370c7b Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Wed, 18 May 2022 14:33:17 -0600 Subject: [PATCH 23/42] docs: fix inputs/outputs tables --- README.md | 62 +++++++++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 1296850..3444df1 100644 --- a/README.md +++ b/README.md @@ -33,40 +33,40 @@ module "humio_logger" { ## Inputs -| Name | Type | Description | Default | -|-------------------------------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| -| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | -| app_name | string | The application name to include in the name of resources created. | | -| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | | -| humio_host | string | The host to ship Humio log/metric events to. | | -| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | | -| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | -| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | false | -| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | -| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | -| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | | -| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | | -| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | | -| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | -| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | +| Name | Type | Description | Default | +|-------------------------------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------| +| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | +| app_name | string | The application name to include in the name of resources created. | | +| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | HTTPS | +| humio_host | string | The host to ship Humio log/metric events to. | cloud.humio.com | +| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | "" | +| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | +| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | true | +| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | +| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | +| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | "" | +| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | [] | +| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | [] | +| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | +| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | v1.2.1 | ## Outputs -| Name | Type | Description | Default | -|-------------------------------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| -| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | -| app_name | string | The application name to include in the name of resources created. | | -| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | | -| humio_host | string | The host to ship Humio log/metric events to. | | -| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | | -| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | -| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | false | -| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | -| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | -| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | | -| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | | -| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | | -| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | +| Name | Type | Description | +|-------------------------------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | +| app_name | string | The application name to include in the name of resources created. | +| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | +| humio_host | string | The host to ship Humio log/metric events to. | +| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | +| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | +| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | +| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | +| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | +| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | +| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | +| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | +| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | | cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | ## Deployment From 8b9721daebcd321364f663e74281f6bb4ef83c4d Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Wed, 18 May 2022 15:58:52 -0600 Subject: [PATCH 24/42] fix: remove unused env variable --- README.md | 5 +---- examples/ci/ci.tf | 3 +-- examples/simple/simple-example.tf | 3 +-- outputs.tf | 4 ---- variables.tf | 5 ----- 5 files changed, 3 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 3444df1..644dae2 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,7 @@ to you. ```hcl module "humio_logger" { source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" - app_env = "dev" - app_name = "humio-logger-ci" + app_name = "humio-logger-ci-dev" humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" } ``` @@ -35,7 +34,6 @@ module "humio_logger" { | Name | Type | Description | Default | |-------------------------------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------| -| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | | app_name | string | The application name to include in the name of resources created. | | | humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | HTTPS | | humio_host | string | The host to ship Humio log/metric events to. | cloud.humio.com | @@ -54,7 +52,6 @@ module "humio_logger" { | Name | Type | Description | |-------------------------------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| app_env | string | The environment of the application. Used to determine what instance of Humio to send log data to. | | app_name | string | The application name to include in the name of resources created. | | humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | | humio_host | string | The host to ship Humio log/metric events to. | diff --git a/examples/ci/ci.tf b/examples/ci/ci.tf index ececa6e..dabc526 100644 --- a/examples/ci/ci.tf +++ b/examples/ci/ci.tf @@ -20,8 +20,7 @@ module "acs" { module "humio_logger" { source = "../.." - app_env = "dev" - app_name = "humio-logger-ci" + app_name = "humio-logger-ci-dev" humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids diff --git a/examples/simple/simple-example.tf b/examples/simple/simple-example.tf index 88b1a01..aacf886 100644 --- a/examples/simple/simple-example.tf +++ b/examples/simple/simple-example.tf @@ -20,8 +20,7 @@ module "acs" { module "humio_logger" { source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" - app_env = "dev" - app_name = "humio-logger-ci" + app_name = "humio-logger-ci-dev" humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids diff --git a/outputs.tf b/outputs.tf index 3381e5e..3a50def 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,7 +1,3 @@ -output "app_env" { - value = var.app_env -} - output "app_name" { value = var.app_name } diff --git a/variables.tf b/variables.tf index d0a96f0..170e9ec 100644 --- a/variables.tf +++ b/variables.tf @@ -1,8 +1,3 @@ -variable "app_env" { - type = string - description = "The environment of the application. Used to determine what instance of Humio to send log data to." -} - variable "app_name" { type = string description = "The application name to include in the name of resources created." From 62db13b943f13f52837bcbee2ec261ed68d177c6 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Wed, 18 May 2022 18:57:27 -0600 Subject: [PATCH 25/42] docs: update readme with typical example --- README.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 644dae2..4ef1e24 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,19 @@ to you. ### Example ```hcl -module "humio_logger" { - source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" - app_name = "humio-logger-ci-dev" - humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" +module "acs" { + source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.5.0" + vpc_vpn_to_campus = true +} + +module "humio_logger" { + source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" + app_name = "humio-logger-ci-dev" + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" + vpc_id = module.acs.vpc.id + subnet_ids = module.acs.private_subnet_ids + humio_host = module.acs.humio_dev_endpoint + humio_ingest_token = module.acs.humio_dev_token } ``` @@ -29,6 +38,7 @@ module "humio_logger" { * Terraform version 0.14 or greater * AWS provider version 3.0 or greater +* (optional) when using the BYU-ACS module ## Inputs From 141daba5f966f5440939bf604a9cd74bb7fa7dbf Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 19 May 2022 11:09:46 -0600 Subject: [PATCH 26/42] fix: add iam capability --- main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.tf b/main.tf index 2a32e1c..b7024b7 100644 --- a/main.tf +++ b/main.tf @@ -32,6 +32,7 @@ resource "aws_security_group" "logging" { resource "aws_cloudformation_stack" "cloudwatch2humio" { name = "${var.app_name}-cloudwatch2humio" template_body = data.local_file.cloudformation.content + capabilities = ["CAPABILITY_IAM"] parameters = { HumioProtocol = var.humio_protocol HumioHost = var.humio_host From a8ed186f1d685d3eabe9f152bedd11dcea674556 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 19 May 2022 16:01:42 -0600 Subject: [PATCH 27/42] fix: add role permissions boundary --- README.md | 7 +++++-- cloudformation.json | 12 +++++++++++- main.tf | 1 + outputs.tf | 4 ++++ variables.tf | 6 ++++++ 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4ef1e24..ac24b95 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,12 @@ module "acs" { module "humio_logger" { source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" app_name = "humio-logger-ci-dev" - humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids humio_host = module.acs.humio_dev_endpoint humio_ingest_token = module.acs.humio_dev_token + humio_lambda_role_permissions_boundary = module.acs.role_permissions_boundary.arn } ``` @@ -49,6 +50,7 @@ module "humio_logger" { | humio_host | string | The host to ship Humio log/metric events to. | cloud.humio.com | | humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | "" | | humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | +| humio_lambda_role_permissions_boundary | string | The ARN of the role permissions boundary to attach to the Humio Lambda role. | "" | | enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | true | | humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | | enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | @@ -67,6 +69,7 @@ module "humio_logger" { | humio_host | string | The host to ship Humio log/metric events to. | | humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | | humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | +| humio_lambda_role_permissions_boundary | string | The ARN of the role permissions boundary to attach to the Humio Lambda role. | | enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | | humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | | enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | @@ -86,5 +89,5 @@ To update to the latest version of Cloudwatch2Humio: 1. Clone the git repository: [https://github.com/humio/cloudwatch2humio](https://github.com/humio/cloudwatch2humio) 2. Copy the `$PROJECT/cloudformation.json` file to this project replacing the old cloudformation template. This file - should not be modified. + should not be modified except to add the PermissionsBoundary to the HumioCloudWatchRole. 3. Ensure that the necessary refactors are implemented in terraform. diff --git a/cloudformation.json b/cloudformation.json index dca9018..6eae3f0 100644 --- a/cloudformation.json +++ b/cloudformation.json @@ -23,6 +23,10 @@ "Description" : "Number of days to retain CloudWatch logs from the Humio Lambda functions.", "Default": 1 }, + "HumioLambdaRolePermissionsBoundary" : { + "Type" : "String", + "Description" : "The ARN of the role permissions boundary to attach to the Humio Lambda role." + }, "EnableCloudWatchLogsAutoSubscription" : { "Type" : "String", "AllowedValues" : [ @@ -83,6 +87,9 @@ }, "ConfigureVPCForIngesterLambdas" : { "Fn::Equals" : [ { "Ref" : "EnableVPCForIngesterLambdas" }, "true" ] + }, + "AddPermissionBoundary" : { + "Fn::Not" : [ { "Fn::Equals" : [ { "Ref" : "HumioLambdaRolePermissionsBoundary" }, "" ] }] } }, "Resources" : { @@ -168,7 +175,10 @@ ] } } - ] + ], + "PermissionsBoundary": { + "Fn::If" : [ "AddPermissionBoundary", { "Ref" : "HumioLambdaRolePermissionsBoundary" }, { "Ref" : "AWS::NoValue" }] + } } }, "HumioCloudWatchLogsIngester" : { diff --git a/main.tf b/main.tf index b7024b7..c54fbf2 100644 --- a/main.tf +++ b/main.tf @@ -38,6 +38,7 @@ resource "aws_cloudformation_stack" "cloudwatch2humio" { HumioHost = var.humio_host HumioIngestToken = var.humio_ingest_token HumioLambdaLogRetention = var.humio_lambda_log_retention + HumioLambdaRolePermissionsBoundary = var.humio_lambda_role_permissions_boundary EnableCloudWatchLogsAutoSubscription = tostring(var.enable_cloudwatch_logs_auto_subscription) HumioCloudWatchLogsSubscriptionPrefix = var.humio_cloudwatch_logs_subscription_prefix EnableCloudWatchLogsBackfillerAutoRun = tostring(var.enable_cloudwatch_logs_backfiller_autorun) diff --git a/outputs.tf b/outputs.tf index 3a50def..5b869e1 100644 --- a/outputs.tf +++ b/outputs.tf @@ -14,6 +14,10 @@ output "humio_lambda_log_retention" { value = var.humio_lambda_log_retention } +output "humio_lambda_role_permissions_boundary" { + value = var.humio_lambda_role_permissions_boundary +} + output "enable_cloudwatch_logs_auto_subscription" { value = var.enable_cloudwatch_logs_auto_subscription } diff --git a/variables.tf b/variables.tf index 170e9ec..3a106b1 100644 --- a/variables.tf +++ b/variables.tf @@ -32,6 +32,12 @@ variable "humio_lambda_log_retention" { default = 1 } +variable "humio_lambda_role_permissions_boundary" { + type = string + description = "The ARN of the role permissions boundary to attach to the Humio Lambda role." + default = "" +} + variable "enable_cloudwatch_logs_auto_subscription" { type = bool description = "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'false' to disable." From 260e3aed3bc0fd974c9081e3ec0f9307289bf15e Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 19 May 2022 17:14:25 -0600 Subject: [PATCH 28/42] docs: add details about subscribing to log groups --- README.md | 49 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ac24b95..db16197 100644 --- a/README.md +++ b/README.md @@ -15,23 +15,48 @@ Carson Mills) so that he can keep an updated list of the various data streams co for them. He will also need to update filters on views to include the new SubIdxNM values so that data will be visible to you. +If you only want specific log groups to be ingested into Humio, you can use the `HumioCloudWatchLogsSubscriber` as this +only subscribes the log ingester to one log group at a time. If you want to subscribe to all log groups available, you +can use the `HumioCloudWatchBackfiller`. If you have set the `EnableCloudWatchLogsBackfillerAutoRun` parameter to true +when creating the stack, then you will _not_ have to manually trigger it as it should already have run on creation and +subscribed the log ingester to all available log groups. Otherwise, both lambdas can be enabled using test events. + +For the `HumioCloudWatchLogsSubscriber` lambda, configure your test event like the example below with “EXAMPLE” +representing an actual log group, and click Test. + +```json +{ + "detail": { + "requestParameters": { + "logGroupName": "EXAMPLE" + } + } +} +``` + +For the `HumioCloudWatchLogsBackfiller` lambda, use the default test event and click Test. This might take a while +depending on the number of log groups that you are subscribing to. + +[For more troubleshooting information, read the documentation.](https://library.humio.com/reference/log-formats/amazon-cloudwatch/#configuring-the-integration) +provided by Humio about this integration. + ### Example ```hcl module "acs" { - source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.5.0" - vpc_vpn_to_campus = true + source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.5.0" + vpc_vpn_to_campus = true } -module "humio_logger" { - source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" - app_name = "humio-logger-ci-dev" - humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" - vpc_id = module.acs.vpc.id - subnet_ids = module.acs.private_subnet_ids - humio_host = module.acs.humio_dev_endpoint - humio_ingest_token = module.acs.humio_dev_token - humio_lambda_role_permissions_boundary = module.acs.role_permissions_boundary.arn +module "humio_logger" { + source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" + app_name = "humio-logger-ci-dev" + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" + vpc_id = module.acs.vpc.id + subnet_ids = module.acs.private_subnet_ids + humio_host = module.acs.humio_dev_endpoint + humio_ingest_token = module.acs.humio_dev_token + humio_lambda_role_permissions_boundary = module.acs.role_permissions_boundary.arn } ``` @@ -39,7 +64,7 @@ module "humio_logger" { * Terraform version 0.14 or greater * AWS provider version 3.0 or greater -* (optional) when using the BYU-ACS module +* BYU-ACS version 3.5.0 or greater ## Inputs From db5eb3b9e8a3c7e4dad8369a0633ddf30b5d4cfd Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Fri, 20 May 2022 09:09:47 -0600 Subject: [PATCH 29/42] docs: remove header in readme --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index db16197..e3ad023 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,6 @@ Infrastructure to watch a CloudWatch log group and forward its logs to Humio. ## Usage -### Prerequisites - Before including this module in your project, be sure to communicate with the Platform Engineer over Humio (currently Carson Mills) so that he can keep an updated list of the various data streams coming into Humio and who is responsible for them. He will also need to update filters on views to include the new SubIdxNM values so that data will be visible From 53f898e979c311f7203fec15b65fdbf60060a8c5 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Fri, 20 May 2022 09:55:58 -0600 Subject: [PATCH 30/42] docs: update humio endpoint --- README.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e3ad023..e8097fc 100644 --- a/README.md +++ b/README.md @@ -46,15 +46,19 @@ module "acs" { vpc_vpn_to_campus = true } -module "humio_logger" { - source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" - app_name = "humio-logger-ci-dev" - humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" - vpc_id = module.acs.vpc.id - subnet_ids = module.acs.private_subnet_ids - humio_host = module.acs.humio_dev_endpoint - humio_ingest_token = module.acs.humio_dev_token - humio_lambda_role_permissions_boundary = module.acs.role_permissions_boundary.arn +locals { + humio_endpoint = "${var.env == "prd" ? module.acs.humio_prd_endpoint : module.acs.humio_dev_endpoint}/api/v1/ingest/humio-structured" +} + +module "cloudwatch2humio" { + source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" + app_name = local.app_name + humio_cloudwatch_logs_subscription_prefix = local.cloudwatch_log_group_name + vpc_id = module.acs.vpc.id + subnet_ids = module.acs.private_subnet_ids + humio_host = local.humio_endpoint + humio_ingest_token = var.env == "prd" ? module.acs.humio_prd_token : module.acs.humio_dev_token + humio_lambda_role_permissions_boundary = module.acs.role_permissions_boundary.arn } ``` From 7fcde7f93a62dc7ff67cb8275294607331f6f51c Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Fri, 20 May 2022 10:04:21 -0600 Subject: [PATCH 31/42] Revert "docs: update humio endpoint" This reverts commit 53f898e979c311f7203fec15b65fdbf60060a8c5. --- README.md | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e8097fc..e3ad023 100644 --- a/README.md +++ b/README.md @@ -46,19 +46,15 @@ module "acs" { vpc_vpn_to_campus = true } -locals { - humio_endpoint = "${var.env == "prd" ? module.acs.humio_prd_endpoint : module.acs.humio_dev_endpoint}/api/v1/ingest/humio-structured" -} - -module "cloudwatch2humio" { - source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" - app_name = local.app_name - humio_cloudwatch_logs_subscription_prefix = local.cloudwatch_log_group_name - vpc_id = module.acs.vpc.id - subnet_ids = module.acs.private_subnet_ids - humio_host = local.humio_endpoint - humio_ingest_token = var.env == "prd" ? module.acs.humio_prd_token : module.acs.humio_dev_token - humio_lambda_role_permissions_boundary = module.acs.role_permissions_boundary.arn +module "humio_logger" { + source = "github.com/byu-oit/terraform-aws-humio-logger?ref=v2.0.0" + app_name = "humio-logger-ci-dev" + humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" + vpc_id = module.acs.vpc.id + subnet_ids = module.acs.private_subnet_ids + humio_host = module.acs.humio_dev_endpoint + humio_ingest_token = module.acs.humio_dev_token + humio_lambda_role_permissions_boundary = module.acs.role_permissions_boundary.arn } ``` From a8d34842af095167131b309b7aacce053418a7a8 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Fri, 20 May 2022 12:29:39 -0600 Subject: [PATCH 32/42] fix: shorten cf stack name --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index c54fbf2..28f90d0 100644 --- a/main.tf +++ b/main.tf @@ -30,7 +30,7 @@ resource "aws_security_group" "logging" { } resource "aws_cloudformation_stack" "cloudwatch2humio" { - name = "${var.app_name}-cloudwatch2humio" + name = var.app_name template_body = data.local_file.cloudformation.content capabilities = ["CAPABILITY_IAM"] parameters = { From c638578ce13a9c8082c7241756b06f4a14cbc5c5 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Fri, 20 May 2022 13:01:36 -0600 Subject: [PATCH 33/42] docs: cleanup old deployment documentation --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e3ad023..8214cb5 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,8 @@ module "humio_logger" { ## Deployment -If you update the Lambda function code, be sure to run `zip -r function.zip .` in the `lambda` folder. +Push changes to the main branch and create a new release of the humio logger +following [the semantic versioning specification](https://semver.org). ## Development From 6eec48dba2615c4c968da0f154d97e5b99263504 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 26 May 2022 07:24:01 -0600 Subject: [PATCH 34/42] docs: add info about ingest tokens --- .github/workflows/ci.yml | 2 +- README.md | 79 ++++++++++++++++--------------- examples/simple/simple-example.tf | 8 +++- 3 files changed, 49 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 522769a..6846430 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: - name: Terraform Plan working-directory: ${{ env.tf_working_dir }} - run: terraform plan.tfplan -input=false -out=plan.tfplan + run: terraform plan.tfplan -var 'humio_dev_token=fake-dev-token' -input=false -out=plan.tfplan - name: Comment Terraform Plan uses: byu-oit/github-action-tf-plan-comment@v1 diff --git a/README.md b/README.md index 8214cb5..ae92587 100644 --- a/README.md +++ b/README.md @@ -2,16 +2,18 @@ # Terraform AWS Humio Logger -Infrastructure to watch a CloudWatch log group and forward its logs to Humio. +Infrastructure to watch a CloudWatch log groups and forward their logs to a Humio log repository. This modules wraps the +[Humio Log Shipper Cloudformation template](https://github.com/humio/cloudwatch2humio). Please file issues in the code +and infrastructure there unless the issue is related to terraform. #### [New to Terraform Modules at BYU?](https://devops.byu.edu/terraform/index.html) ## Usage -Before including this module in your project, be sure to communicate with the Platform Engineer over Humio (currently -Carson Mills) so that he can keep an updated list of the various data streams coming into Humio and who is responsible -for them. He will also need to update filters on views to include the new SubIdxNM values so that data will be visible -to you. +Before including this module in your project, you must create a new humio log repository and generate an ingest token. +The default token will work, but you should specify a parser for the logs. +Read [Ingest Tokens documentation](https://library.humio.com/cloud/docs/ingesting-data/ingest-tokens/) for more +information. If you only want specific log groups to be ingested into Humio, you can use the `HumioCloudWatchLogsSubscriber` as this only subscribes the log ingester to one log group at a time. If you want to subscribe to all log groups available, you @@ -52,8 +54,9 @@ module "humio_logger" { humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids + humio_protocol = "HTTP" # Only the http protocol is supported for the ACS-provided Humio Endpoints (as of May 26, 2022) humio_host = module.acs.humio_dev_endpoint - humio_ingest_token = module.acs.humio_dev_token + humio_ingest_token = var.humio_dev_token # Must provide this for each humio log repo humio_lambda_role_permissions_boundary = module.acs.role_permissions_boundary.arn } ``` @@ -66,41 +69,41 @@ module "humio_logger" { ## Inputs -| Name | Type | Description | Default | -|-------------------------------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------| -| app_name | string | The application name to include in the name of resources created. | | -| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | HTTPS | -| humio_host | string | The host to ship Humio log/metric events to. | cloud.humio.com | -| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | "" | -| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | -| humio_lambda_role_permissions_boundary | string | The ARN of the role permissions boundary to attach to the Humio Lambda role. | "" | -| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | true | -| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | -| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | -| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | "" | -| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | [] | -| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | [] | -| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | -| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | v1.2.1 | +| Name | Type | Description | Default | +|-------------------------------------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------|-----------------| +| app_name | string | The application name to include in the name of resources created. | | +| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | HTTPS | +| humio_host | string | The host to ship Humio log/metric events to. | cloud.humio.com | +| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | "" | +| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | +| humio_lambda_role_permissions_boundary | string | The ARN of the role permissions boundary to attach to the Humio Lambda role. | "" | +| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | true | +| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | +| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | +| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | "" | +| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | [] | +| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | [] | +| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | +| cloudwatch2humio_version | string | The version of the integration to be installed. Available releases can be found under releases in the GitHub repository. | v1.2.1 | ## Outputs -| Name | Type | Description | -|-------------------------------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| app_name | string | The application name to include in the name of resources created. | -| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | -| humio_host | string | The host to ship Humio log/metric events to. | -| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | -| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | -| humio_lambda_role_permissions_boundary | string | The ARN of the role permissions boundary to attach to the Humio Lambda role. | -| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | -| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | -| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | -| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | -| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | -| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | -| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | -| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack, the default value is the newest version. Available releases can be found under releases in the GitHub repository. | +| Name | Type | Description | +|-------------------------------------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| app_name | string | The application name to include in the name of resources created. | +| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | +| humio_host | string | The host to ship Humio log/metric events to. | +| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | +| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | +| humio_lambda_role_permissions_boundary | string | The ARN of the role permissions boundary to attach to the Humio Lambda role. | +| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | +| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | +| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | +| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | +| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | +| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | +| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | +| cloudwatch2humio_version | string | The version of the integration to be installed. When creating a new stack. | ## Deployment diff --git a/examples/simple/simple-example.tf b/examples/simple/simple-example.tf index aacf886..606015e 100644 --- a/examples/simple/simple-example.tf +++ b/examples/simple/simple-example.tf @@ -13,6 +13,12 @@ provider "aws" { region = "us-west-2" } +variable "humio_dev_token" { + type = string + description = "The Humio ingest token to use for shipping logs to Humio from the cloudwatch ingest lambda." + sensitive = true +} + module "acs" { source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.5.0" vpc_vpn_to_campus = true @@ -25,5 +31,5 @@ module "humio_logger" { vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids humio_host = module.acs.humio_dev_endpoint - humio_ingest_token = module.acs.humio_dev_token + humio_ingest_token = var.humio_dev_token } From f750ad6301a502c7b4913d256668b61468c812cb Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 26 May 2022 07:44:39 -0600 Subject: [PATCH 35/42] fi: add variable to ci example --- README.md | 11 ++++++++--- examples/ci/ci.tf | 11 +++++++++-- examples/simple/simple-example.tf | 7 ++++--- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ae92587..86e1f81 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,16 @@ and infrastructure there unless the issue is related to terraform. ## Usage -Before including this module in your project, you must create a new humio log repository and generate an ingest token. -The default token will work, but you should specify a parser for the logs. -Read [Ingest Tokens documentation](https://library.humio.com/cloud/docs/ingesting-data/ingest-tokens/) for more +Before including this module in your project, you should create a new humio log repository and generate an ingest token +to pass into this module. +Read the [Ingest Tokens documentation](https://library.humio.com/cloud/docs/ingesting-data/ingest-tokens/) for more information. +> #### ACS Humio Endpoints +> The [ACS Info Terraform Module](https://github.com/byu-oit/terraform-aws-acs-info) provides the Humio dev and prd +> endpoints for convenience. It is recommended that you only use the prd endpoint (even for deployments of dev +> resources) because it is a stable endpoint. + If you only want specific log groups to be ingested into Humio, you can use the `HumioCloudWatchLogsSubscriber` as this only subscribes the log ingester to one log group at a time. If you want to subscribe to all log groups available, you can use the `HumioCloudWatchBackfiller`. If you have set the `EnableCloudWatchLogsBackfillerAutoRun` parameter to true diff --git a/examples/ci/ci.tf b/examples/ci/ci.tf index dabc526..708dffd 100644 --- a/examples/ci/ci.tf +++ b/examples/ci/ci.tf @@ -13,6 +13,12 @@ provider "aws" { region = "us-west-2" } +variable "humio_token" { + type = string + description = "The Humio ingest token to use for shipping logs to Humio from the cloudwatch ingest lambda." + sensitive = true +} + module "acs" { source = "github.com/byu-oit/terraform-aws-acs-info?ref=v3.5.0" vpc_vpn_to_campus = true @@ -24,6 +30,7 @@ module "humio_logger" { humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids - humio_host = module.acs.humio_dev_endpoint - humio_ingest_token = module.acs.humio_dev_token + humio_protocol = "HTTP" + humio_host = module.acs.humio_prd_endpoint + humio_ingest_token = var.humio_token } diff --git a/examples/simple/simple-example.tf b/examples/simple/simple-example.tf index 606015e..8093df8 100644 --- a/examples/simple/simple-example.tf +++ b/examples/simple/simple-example.tf @@ -13,7 +13,7 @@ provider "aws" { region = "us-west-2" } -variable "humio_dev_token" { +variable "humio_token" { type = string description = "The Humio ingest token to use for shipping logs to Humio from the cloudwatch ingest lambda." sensitive = true @@ -30,6 +30,7 @@ module "humio_logger" { humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids - humio_host = module.acs.humio_dev_endpoint - humio_ingest_token = var.humio_dev_token + humio_protocol = "HTTP" + humio_host = module.acs.humio_prd_endpoint + humio_ingest_token = var.humio_token } From 6fa4ada6f50bf374da84883cf4e48f44a6900470 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 26 May 2022 07:49:37 -0600 Subject: [PATCH 36/42] ci: fix example plan command --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6846430..cc68b06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: - name: Terraform Plan working-directory: ${{ env.tf_working_dir }} - run: terraform plan.tfplan -var 'humio_dev_token=fake-dev-token' -input=false -out=plan.tfplan + run: terraform plan.tfplan -var 'humio_token=fake-token' -input=false -out=plan.tfplan - name: Comment Terraform Plan uses: byu-oit/github-action-tf-plan-comment@v1 From 63485716b92b7ae61421c1e145514650255b67bc Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 2 Jun 2022 12:29:53 -0600 Subject: [PATCH 37/42] docs: update the host port --- README.md | 2 +- examples/ci/ci.tf | 2 +- examples/simple/simple-example.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 86e1f81..57d83b9 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ module "humio_logger" { vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids humio_protocol = "HTTP" # Only the http protocol is supported for the ACS-provided Humio Endpoints (as of May 26, 2022) - humio_host = module.acs.humio_dev_endpoint + humio_host = "${module.acs.humio_prd_endpoint}:8080" # Default is port 80 humio_ingest_token = var.humio_dev_token # Must provide this for each humio log repo humio_lambda_role_permissions_boundary = module.acs.role_permissions_boundary.arn } diff --git a/examples/ci/ci.tf b/examples/ci/ci.tf index 708dffd..35b875e 100644 --- a/examples/ci/ci.tf +++ b/examples/ci/ci.tf @@ -31,6 +31,6 @@ module "humio_logger" { vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids humio_protocol = "HTTP" - humio_host = module.acs.humio_prd_endpoint + humio_host = "${module.acs.humio_prd_endpoint}:8080" humio_ingest_token = var.humio_token } diff --git a/examples/simple/simple-example.tf b/examples/simple/simple-example.tf index 8093df8..5de776b 100644 --- a/examples/simple/simple-example.tf +++ b/examples/simple/simple-example.tf @@ -31,6 +31,6 @@ module "humio_logger" { vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids humio_protocol = "HTTP" - humio_host = module.acs.humio_prd_endpoint + humio_host = "${module.acs.humio_prd_endpoint}:8080" humio_ingest_token = var.humio_token } From eac3e911d8c21b6c100d8f841e0be5b8767e2b55 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 2 Jun 2022 13:10:08 -0600 Subject: [PATCH 38/42] fix: remove cloudtrail resources --- cloudformation.json | 65 --------------------------------------------- 1 file changed, 65 deletions(-) diff --git a/cloudformation.json b/cloudformation.json index 6eae3f0..fdc75f9 100644 --- a/cloudformation.json +++ b/cloudformation.json @@ -372,71 +372,6 @@ "StackName" : { "Ref" : "AWS::StackName" } } }, - "HumioCloudWatchLogsSubscriberS3Bucket" : { - "Condition" : "CreateAutoSubscriptionResources", - "Type" : "AWS::S3::Bucket", - "Properties" : { - "AccessControl" : "BucketOwnerFullControl", - "BucketName" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", "cloudtrail" ] ] - } - } - }, - "HumioCloudWatchLogsSubscriberS3BucketPolicy" : { - "Condition" : "CreateAutoSubscriptionResources", - "DependsOn" : [ "HumioCloudWatchLogsSubscriberS3Bucket" ], - "Type" : "AWS::S3::BucketPolicy", - "Properties" : { - "Bucket" : { "Ref" : "HumioCloudWatchLogsSubscriberS3Bucket" }, - "PolicyDocument" : { - "Version" : "2012-10-17", - "Statement" : [ - { - "Sid" : "AWSCloudTrailAclCheck20150319", - "Effect" : "Allow", - "Principal" : { - "Service" : "cloudtrail.amazonaws.com" - }, - "Action" : "s3:GetBucketAcl", - "Resource" : { - "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberS3Bucket", "Arn" ] - } - }, - { - "Sid" : "AWSCloudTrailWrite20150319", - "Effect" : "Allow", - "Principal" : { - "Service" : "cloudtrail.amazonaws.com" - }, - "Action" : "s3:PutObject", - "Resource" : { - "Fn::Join" : [ "", [ { "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberS3Bucket", "Arn" ] }, "/AWSLogs/", { "Ref" : "AWS::AccountId" }, "/*" ] ] - }, - "Condition" : { - "StringEquals" : { "s3:x-amz-acl" : "bucket-owner-full-control" } - } - } - ] - } - } - }, - "HumioCloudWatchLogsSubscriberCloudTrail" : { - "Condition" : "CreateAutoSubscriptionResources", - "DependsOn" : [ "HumioCloudWatchLogsSubscriberS3BucketPolicy" ], - "Type" : "AWS::CloudTrail::Trail", - "Properties" : { - "EnableLogFileValidation" : false, - "IncludeGlobalServiceEvents" : true, - "IsMultiRegionTrail" : true, - "IsLogging" : true, - "S3BucketName" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", "cloudtrail" ] ] - }, - "TrailName" : { - "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", { "Ref" : "AWS::AccountId" } ] ] - } - } - }, "HumioCloudWatchLogsSubscriberEventRule" : { "Condition" : "CreateAutoSubscriptionResources", "DependsOn" : [ "HumioCloudWatchLogsSubscriber" ], From 9540296f14ce64820eae7bb9144946136c6ab1cc Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 9 Jun 2022 07:33:25 -0600 Subject: [PATCH 39/42] docs: clarify documentation --- README.md | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 57d83b9..5e71ccb 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@ # Terraform AWS Humio Logger -Infrastructure to watch a CloudWatch log groups and forward their logs to a Humio log repository. This modules wraps the -[Humio Log Shipper Cloudformation template](https://github.com/humio/cloudwatch2humio). Please file issues in the code -and infrastructure there unless the issue is related to terraform. +Infrastructure to watch a CloudWatch log groups and forward their logs to a Humio log repository. This modules wraps the +[Humio Log Shipper Cloudformation template](https://github.com/humio/cloudwatch2humio/cloudformation-no-trail.json). *It +does not create CloudTrail resources.* Please file issues in the code and infrastructure there unless the issue is +related to terraform. #### [New to Terraform Modules at BYU?](https://devops.byu.edu/terraform/index.html) @@ -15,7 +16,6 @@ to pass into this module. Read the [Ingest Tokens documentation](https://library.humio.com/cloud/docs/ingesting-data/ingest-tokens/) for more information. -> #### ACS Humio Endpoints > The [ACS Info Terraform Module](https://github.com/byu-oit/terraform-aws-acs-info) provides the Humio dev and prd > endpoints for convenience. It is recommended that you only use the prd endpoint (even for deployments of dev > resources) because it is a stable endpoint. @@ -59,10 +59,13 @@ module "humio_logger" { humio_cloudwatch_logs_subscription_prefix = "/humio-logger-ci/dev" vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids - humio_protocol = "HTTP" # Only the http protocol is supported for the ACS-provided Humio Endpoints (as of May 26, 2022) + humio_protocol = "HTTP" + # Only the http protocol is supported for the ACS-provided Humio Endpoints (as of May 26, 2022) humio_host = "${module.acs.humio_prd_endpoint}:8080" # Default is port 80 humio_ingest_token = var.humio_dev_token # Must provide this for each humio log repo humio_lambda_role_permissions_boundary = module.acs.role_permissions_boundary.arn + enable_cloudwatch_logs_auto_subscription = true + enable_cloudwatch_logs_backfiller_autorun = true } ``` @@ -74,22 +77,22 @@ module "humio_logger" { ## Inputs -| Name | Type | Description | Default | -|-------------------------------------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------|-----------------| -| app_name | string | The application name to include in the name of resources created. | | -| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | HTTPS | -| humio_host | string | The host to ship Humio log/metric events to. | cloud.humio.com | -| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | "" | -| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | -| humio_lambda_role_permissions_boundary | string | The ARN of the role permissions boundary to attach to the Humio Lambda role. | "" | -| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | true | -| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | -| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. | false | -| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | "" | -| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | [] | -| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | [] | -| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | -| cloudwatch2humio_version | string | The version of the integration to be installed. Available releases can be found under releases in the GitHub repository. | v1.2.1 | +| Name | Type | Description | Default | +|-------------------------------------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------| +| app_name | string | The application name to include in the name of resources created. | | +| humio_protocol | string | The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended. | HTTPS | +| humio_host | string | The host to ship Humio log/metric events to. | cloud.humio.com | +| humio_ingest_token | string | The value of the ingest token for the repository from your Humio account to ship log/metric events to. | "" | +| humio_lambda_log_retention | number | Number of days to retain CloudWatch logs from the Humio Lambda functions. | 1 | +| humio_lambda_role_permissions_boundary | string | The ARN of the role permissions boundary to attach to the Humio Lambda role. | "" | +| enable_cloudwatch_logs_auto_subscription | bool | Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable. | true | +| humio_cloudwatch_logs_subscription_prefix | string | Humio will only subscribe to log groups with the prefix specified. | "" | +| enable_cloudwatch_logs_backfiller_autorun | bool | Make the backfiller run automatically when created. Set to 'true' to enable. Enabling the backfiller will [delete existing subscriptions](https://github.com/humio/cloudwatch2humio/issues/31). | false | +| vpc_id | string | Use a VPC for the lambda ingester functions. Pass in a vpc to enable. | "" | +| security_group_ids | list(string) | A list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled. | [] | +| subnet_ids | list(string) | A list of subnet ids used by the VPC configuration that the ingester lambda functions will be deployed into. Only required if VPC is enabled. | [] | +| humio_lambda_log_level | string | The log level for the Humio lambdas. (DEBUG, INFO, WARNING, ERROR, CRITICAL) | "INFO" | +| cloudwatch2humio_version | string | The version of the integration to be installed. Available releases can be found under releases in the GitHub repository. | v1.2.1 | ## Outputs From 4de766ac9bb26207e419f6154da3612ec100eb96 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 9 Jun 2022 07:46:03 -0600 Subject: [PATCH 40/42] ci: test on terraform 1.2.x --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc68b06..aa5d5fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - terraform: ['0.14.x', '0.15.x', '1.0.x', '1.1.x'] + terraform: ['0.14.x', '0.15.x', '1.0.x', '1.1.x', '1.2.x'] fail-fast: false steps: - uses: actions/checkout@v2 From ac10a7dab419da74e7bf665fe593271d307ac406 Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 23 Jun 2022 15:10:52 -0600 Subject: [PATCH 41/42] ci: remove dependecy on an acs variable --- examples/ci/ci.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/ci/ci.tf b/examples/ci/ci.tf index 35b875e..95c37a8 100644 --- a/examples/ci/ci.tf +++ b/examples/ci/ci.tf @@ -31,6 +31,7 @@ module "humio_logger" { vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids humio_protocol = "HTTP" - humio_host = "${module.acs.humio_prd_endpoint}:8080" + # humio_host = "${module.acs.humio_prd_endpoint}:8080" + humio_host = "example.com" humio_ingest_token = var.humio_token } From ad179d7c3ed4973f8ad7a471a1b24be43e53e9ab Mon Sep 17 00:00:00 2001 From: Spencer Tuft Date: Thu, 23 Jun 2022 15:13:54 -0600 Subject: [PATCH 42/42] style: remove comment to fix linting --- examples/ci/ci.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/ci/ci.tf b/examples/ci/ci.tf index 95c37a8..49c2c9b 100644 --- a/examples/ci/ci.tf +++ b/examples/ci/ci.tf @@ -31,7 +31,6 @@ module "humio_logger" { vpc_id = module.acs.vpc.id subnet_ids = module.acs.private_subnet_ids humio_protocol = "HTTP" - # humio_host = "${module.acs.humio_prd_endpoint}:8080" humio_host = "example.com" humio_ingest_token = var.humio_token }