diff --git a/README.md b/README.md index ef580c7..7dc230f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ + Appvia Banner

Terraform Registry Latest Release Slack Community Contributors @@ -35,7 +36,7 @@ module "cudos_framework" { enable_sso = true enable_tao_dashboard = false saml_metadata = file("${path.module}/assets/saml-metadata.xml") - quicksights_username = var.quicksights_username + quicksight_dashboard_owner = var.quicksight_dashboard_owner tags = var.tags providers = { diff --git a/assets/cloudformation/cudos/data-exports-aggregation.yaml b/assets/cloudformation/cudos/data-exports-aggregation.yaml index df97edb..5eb95a9 100644 --- a/assets/cloudformation/cudos/data-exports-aggregation.yaml +++ b/assets/cloudformation/cudos/data-exports-aggregation.yaml @@ -1,12 +1,11 @@ -# -## https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/refs/heads/main/cfn-templates/data-exports-aggregation.yaml -# AWSTemplateFormatVersion: "2010-09-09" -Description: AWS Billing Data Export Aggregation v0.1.5 +Description: AWS Billing Data Export Aggregation v0.2.0 Metadata: + AWS::CloudFormation::Interface: ParameterGroups: - - Label: + - + Label: default: "Common Parameters Configuration" Parameters: - DestinationAccountId @@ -14,11 +13,13 @@ Metadata: - ManageCUR2 - ManageFOCUS - ManageCOH - - Label: + - + Label: default: "Parameters needed in Destination (Data Collection) Account only" Parameters: - SourceAccountIds - - Label: + - + Label: default: "Technical Parameters. Please do not change." Parameters: - EnableSCAD @@ -42,8 +43,11 @@ Metadata: default: "Enable Split Cost Allocation Data (SCAD) in CUR 2.0" TimeGranularity: default: "CUR 2.0 Granularity. Do not change." + LakeFormationEnabled: + default: "I have LakeFormation permission model in place for this account & my CFN deployment credentials have administrative rights on LakeFormation" Parameters: + ## # Common params ## @@ -55,22 +59,22 @@ Parameters: Type: String Default: "cid" Description: "Prefix used for all named resources, including S3 Bucket. Must be the same in destination and source stacks" - MaxLength: 37 # = 63 - len('-123456789012-data-exports') + MaxLength: 37 # = 63 - len('-123456789012-data-exports') AllowedPattern: "^[a-z0-9]+[a-z0-9-]{1,61}[a-z0-9]+$" ManageCUR2: Type: String Description: "" - AllowedValues: ["yes", "no"] + AllowedValues: ['yes', 'no'] Default: "no" ManageFOCUS: Type: String Description: "NOTE: you can have only one export of this type" - AllowedValues: ["yes", "no"] + AllowedValues: ['yes', 'no'] Default: "no" ManageCOH: Type: String - Description: "NOTE: you must have Cost Optimization Hub Enabled" - AllowedValues: ["yes", "no"] + Description: "NOTE: you must have Cost Optimization Hub Enabled" + AllowedValues: ['yes', 'no'] Default: "no" ## @@ -79,7 +83,7 @@ Parameters: SourceAccountIds: Type: String AllowedPattern: "^((\\d{12})\\,?)*$" - Default: "" + Default: '' Description: "Ex: 12345678912,98745612312,.... If you install all in the same account (Source=Destination) please put Destination Account Id first in the list. " ## @@ -88,107 +92,76 @@ Parameters: EnableSCAD: Type: String Description: Whether to enable Split Cost Allocation Data (Scad). Set this to 'No', if you experience performance issues due to dataset size. - AllowedValues: ["yes", "no"] + AllowedValues: ['yes', 'no'] Default: "yes" RolePath: Type: String Description: Path for roles where PermissionBoundaries can limit location - Default: "/" + Default: '/' TimeGranularity: Type: String Description: Changing of this parameter will require redeployment of this Stack, purging of data in Destination and then additional Backfill request. HOURLY is a recommended option unless your AWS invoice is more then $50M (in this case contact your TAM when installing). Default: "HOURLY" AllowedValues: ["HOURLY", "DAILY", "MONTHLY"] + LakeFormationEnabled: + Type: String + Description: Choose 'yes' if Lake Formation permission model is in place for the account. If you are not sure, leave it as 'no'. You need to install cid-lakeformation-prerequisite.yaml first. + Default: "no" + AllowedValues: ["yes", "no"] Conditions: - EmptySourceAccountIds: !Equals [!Ref SourceAccountIds, ""] - IsDestinationAccount: - !Equals [!Ref DestinationAccountId, !Ref "AWS::AccountId"] + EmptySourceAccountIds: !Equals [ !Ref SourceAccountIds, ''] + IsDestinationAccount: !Equals [!Ref DestinationAccountId, !Ref 'AWS::AccountId'] IsSourceAccount: # it is Source account if it is not a destination or if it is a destination and it is listed in Source Accounts (as the list one). # Unfortunately, there no 'Fn::Contains' in Conditions, so we need to request user setting Dest account as the first. Fn::Or: - !Not [!Condition IsDestinationAccount] - - !Equals [ - !Ref "AWS::AccountId", - !Select [0, !Split [",", !Sub "${SourceAccountIds},"]], - ] + - !Equals [!Ref 'AWS::AccountId', !Select [0, !Split [',', !Sub '${SourceAccountIds},']]] RegionSupportsDataExportsViaCFN: # CFN supports DataExports only in us-east-1 and cn-northwest-1. Other regions must use lambda. Fn::Or: - - !Equals [!Ref "AWS::Region", "us-east-1"] - - !Equals [!Ref "AWS::Region", "cn-northwest-1"] - ManageCUR2: !Equals [!Ref ManageCUR2, "yes"] - ManageFOCUS: !Equals [!Ref ManageFOCUS, "yes"] - ManageCOH: !Equals [!Ref ManageCOH, "yes"] - EnableSCAD: !Equals [!Ref EnableSCAD, "yes"] + - !Equals [!Ref 'AWS::Region', 'us-east-1'] + - !Equals [!Ref 'AWS::Region', 'cn-northwest-1'] + ManageCUR2: !Equals [!Ref ManageCUR2, 'yes'] + ManageFOCUS: !Equals [!Ref ManageFOCUS, 'yes'] + ManageCOH: !Equals [!Ref ManageCOH, 'yes'] + EnableSCAD: !Equals [!Ref EnableSCAD, 'yes'] DeployDataExport: Fn::Or: - !Condition ManageCUR2 - !Condition ManageFOCUS - !Condition ManageCOH - DeployCOHServiceRole: !And [!Condition IsSourceAccount, !Condition ManageCOH] - DeployCUR2ViaCFN: - !And [ - !Condition IsSourceAccount, - !Condition ManageCUR2, - !Condition RegionSupportsDataExportsViaCFN, - ] - DeployFOCUSViaCFN: - !And [ - !Condition IsSourceAccount, - !Condition ManageFOCUS, - !Condition RegionSupportsDataExportsViaCFN, - ] - DeployCOHViaCFN: - !And [ - !Condition IsSourceAccount, - !Condition ManageCOH, - !Condition RegionSupportsDataExportsViaCFN, - ] - DeployCUR2ViaLambda: - !And [ - !Condition IsSourceAccount, - !Condition ManageCUR2, - !Not [!Condition RegionSupportsDataExportsViaCFN], - ] - DeployFOCUSViaLambda: - !And [ - !Condition IsSourceAccount, - !Condition ManageFOCUS, - !Not [!Condition RegionSupportsDataExportsViaCFN], - ] - DeployCOHViaLambda: - !And [ - !Condition IsSourceAccount, - !Condition ManageCOH, - !Not [!Condition RegionSupportsDataExportsViaCFN], - ] - DeployCUR2Table: !And [!Condition IsDestinationAccount, !Condition ManageCUR2] - DeployFOCUSTable: - !And [!Condition IsDestinationAccount, !Condition ManageFOCUS] - DeployCOHTable: !And [!Condition IsDestinationAccount, !Condition ManageCOH] - DeployAnyExportViaLambda: - !Or [ - !Condition DeployCUR2ViaLambda, - !Condition DeployFOCUSViaLambda, - !Condition DeployCOHViaLambda, - ] - DeployAnyTable: - !Or [ - !Condition DeployFOCUSTable, - !Condition DeployCUR2Table, - !Condition DeployCOHTable, - ] + DeploySourceS3: + Fn::And: + - !Condition IsSourceAccount + - !Condition DeployDataExport + DeployCOHServiceRole: !And [!Condition IsSourceAccount, !Condition ManageCOH] + DeployCUR2ViaCFN: !And [!Condition IsSourceAccount, !Condition ManageCUR2, !Condition RegionSupportsDataExportsViaCFN] + DeployFOCUSViaCFN: !And [!Condition IsSourceAccount, !Condition ManageFOCUS, !Condition RegionSupportsDataExportsViaCFN] + DeployCOHViaCFN: !And [!Condition IsSourceAccount, !Condition ManageCOH, !Condition RegionSupportsDataExportsViaCFN] + DeployCUR2ViaLambda: !And [!Condition IsSourceAccount, !Condition ManageCUR2, !Not [!Condition RegionSupportsDataExportsViaCFN]] + DeployFOCUSViaLambda: !And [!Condition IsSourceAccount, !Condition ManageFOCUS, !Not [!Condition RegionSupportsDataExportsViaCFN]] + DeployCOHViaLambda: !And [!Condition IsSourceAccount, !Condition ManageCOH, !Not [!Condition RegionSupportsDataExportsViaCFN]] + DeployCUR2Table: !And [!Condition IsDestinationAccount, !Condition ManageCUR2] + DeployFOCUSTable: !And [!Condition IsDestinationAccount, !Condition ManageFOCUS] + DeployCOHTable: !And [!Condition IsDestinationAccount, !Condition ManageCOH] + DeployAnyExportViaLambda: !Or [!Condition DeployCUR2ViaLambda, !Condition DeployFOCUSViaLambda, !Condition DeployCOHViaLambda] + DeployAnyTable: !Or [!Condition DeployFOCUSTable, !Condition DeployCUR2Table, !Condition DeployCOHTable] + + NeedLakeFormationEnabledDB: !And [ !Equals [ !Ref LakeFormationEnabled, "yes"], !Condition DeployAnyTable] + NeedLakeFormationEnabledCOH: !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageCOH] + NeedLakeFormationEnabledCUR2: !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageCUR2] + NeedLakeFormationEnabledFOCUS: !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageFOCUS] Mappings: DataExports: #Mappings for storing values for different Data Exports tables CUR2: DefaultQuery: >- - SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment + SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment FROM COST_AND_USAGE_REPORT SCADQuery: >- - SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment, split_line_item_actual_usage, split_line_item_net_split_cost, split_line_item_net_unused_cost, split_line_item_parent_resource_id, split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, split_line_item_split_usage_ratio, split_line_item_unused_cost + SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment, split_line_item_actual_usage, split_line_item_net_split_cost, split_line_item_net_unused_cost, split_line_item_parent_resource_id, split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, split_line_item_split_usage_ratio, split_line_item_unused_cost FROM COST_AND_USAGE_REPORT FOCUS: DefaultQuery: >- @@ -200,9 +173,11 @@ Mappings: FROM COST_OPTIMIZATION_RECOMMENDATIONS Resources: - ########################################################################### - # Destination Account Resources - ########################################################################### + +########################################################################### +# Destination Account Resources +########################################################################### + DestinationS3: Type: AWS::S3::Bucket @@ -243,7 +218,7 @@ Resources: Metadata: cfn_nag: rules_to_suppress: - - id: "W35" + - id: 'W35' reason: "Data buckets would generate too much logs" cfn-lint: config: @@ -251,7 +226,7 @@ Resources: - W3045 # Need to use AccessControl for replication DestinationS3BucketPolicy: - Type: "AWS::S3::BucketPolicy" + Type: 'AWS::S3::BucketPolicy' Condition: IsDestinationAccount DeletionPolicy: Retain UpdateReplacePolicy: Delete @@ -267,8 +242,8 @@ Resources: Principal: "*" Action: s3:* Resource: - - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" - - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" + - !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}' + - !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}/*' Condition: NumericLessThan: s3:TlsVersion: 1.2 @@ -277,8 +252,8 @@ Resources: Principal: "*" Action: s3:* Resource: - - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" - - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" + - !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}' + - !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}/*' Condition: Bool: aws:SecureTransport: false @@ -289,11 +264,11 @@ Resources: Fn::If: - EmptySourceAccountIds - !Ref AWS::AccountId - - !Split [",", !Ref SourceAccountIds] + - !Split [',', !Ref SourceAccountIds] Action: - s3:ReplicateDelete - s3:ReplicateObject - Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" + Resource: !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}/*' - Sid: AllowReplicationRead Effect: Allow Principal: @@ -301,17 +276,17 @@ Resources: Fn::If: - EmptySourceAccountIds - !Ref AWS::AccountId - - !Split [",", !Ref SourceAccountIds] + - !Split [',', !Ref SourceAccountIds] Action: - s3:ListBucket - s3:ListBucketVersions - s3:GetBucketVersioning - s3:PutBucketVersioning - Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" + Resource: !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}' SourceS3: Type: AWS::S3::Bucket - Condition: DeployDataExport + Condition: DeploySourceS3 DeletionPolicy: Delete UpdateReplacePolicy: Delete Properties: @@ -347,7 +322,7 @@ Resources: Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports" StorageClass: STANDARD Id: ReplicateCUR2Data - Prefix: !Sub "cur2/${AWS::AccountId}/${ResourcePrefix}-cur2/data/" # Hardcoded export name + Prefix: !Sub "cur2/${AWS::AccountId}/${ResourcePrefix}-cur2/data/" # Hardcoded export name Status: Enabled - Destination: Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports" @@ -367,10 +342,13 @@ Resources: Status: Enabled NoncurrentVersionExpirationInDays: 32 # 1 month ExpirationInDays: 64 # 2 months + Tags: # Hacky way to manage conditional dependencies + - Key: IgnoreConditionalDependency + Value: !If [IsDestinationAccount, !Ref DestinationS3, ''] Metadata: cfn_nag: rules_to_suppress: - - id: "W35" + - id: 'W35' reason: "Data buckets would generate too much logs" cfn-lint: config: @@ -378,8 +356,8 @@ Resources: - W3045 # Need to use AccessControl for replication SourceS3BucketPolicy: - Type: "AWS::S3::BucketPolicy" - Condition: DeployDataExport + Type: 'AWS::S3::BucketPolicy' + Condition: DeploySourceS3 DeletionPolicy: Delete UpdateReplacePolicy: Delete Properties: @@ -393,8 +371,8 @@ Resources: Principal: "*" Action: s3:* Resource: - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}" - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}/*" + - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}' + - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}/*' Condition: NumericLessThan: s3:TlsVersion: 1.2 @@ -403,8 +381,8 @@ Resources: Principal: "*" Action: s3:* Resource: - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}" - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}/*" + - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}' + - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}/*' Condition: Bool: aws:SecureTransport: false @@ -417,21 +395,22 @@ Resources: - s3:GetBucketPolicy - s3:PutObject Resource: - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}" - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}/*" + - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}' + - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}/*' Condition: StringEquals: aws:SourceAccount: !Ref AWS::AccountId ReplicationRole: - Condition: DeployDataExport + Condition: DeploySourceS3 Type: AWS::IAM::Role Properties: Path: !Sub /${ResourcePrefix}/ AssumeRolePolicyDocument: - Version: "2012-10-17" + Version: '2012-10-17' Statement: - - Effect: Allow + - + Effect: Allow Principal: Service: - "s3.amazonaws.com" @@ -471,19 +450,13 @@ Resources: Properties: Export: DataQuery: - QueryStatement: - !If [ - EnableSCAD, - !FindInMap [DataExports, CUR2, SCADQuery], - !FindInMap [DataExports, CUR2, DefaultQuery], - ] + QueryStatement: !If [EnableSCAD, !FindInMap [DataExports, CUR2, SCADQuery], !FindInMap [DataExports, CUR2, DefaultQuery]] TableConfigurations: COST_AND_USAGE_REPORT: TIME_GRANULARITY: !Ref TimeGranularity INCLUDE_RESOURCES: "TRUE" INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY: "FALSE" - INCLUDE_SPLIT_COST_ALLOCATION_DATA: - !If [EnableSCAD, "TRUE", "FALSE"] + INCLUDE_SPLIT_COST_ALLOCATION_DATA: !If [EnableSCAD, "TRUE", "FALSE"] Description: "CUR 2.0 export for aggregation in CID" DestinationConfigurations: S3Destination: @@ -495,7 +468,7 @@ Resources: Format: "PARQUET" Compression: "PARQUET" OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-cur2" + Name: !Sub '${ResourcePrefix}-cur2' RefreshCadence: Frequency: "SYNCHRONOUS" @@ -508,19 +481,13 @@ Resources: BucketPolicyWait: !Ref SourceS3BucketPolicy Export: DataQuery: - QueryStatement: - !If [ - EnableSCAD, - !FindInMap [DataExports, CUR2, SCADQuery], - !FindInMap [DataExports, CUR2, DefaultQuery], - ] + QueryStatement: !If [EnableSCAD, !FindInMap [DataExports, CUR2, SCADQuery], !FindInMap [DataExports, CUR2, DefaultQuery]] TableConfigurations: COST_AND_USAGE_REPORT: TIME_GRANULARITY: !Ref TimeGranularity INCLUDE_RESOURCES: "TRUE" INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY: "FALSE" - INCLUDE_SPLIT_COST_ALLOCATION_DATA: - !If [EnableSCAD, "TRUE", "FALSE"] + INCLUDE_SPLIT_COST_ALLOCATION_DATA: !If [EnableSCAD, "TRUE", "FALSE"] Description: "CUR 2.0 export for aggregation in CID" DestinationConfigurations: S3Destination: @@ -532,7 +499,7 @@ Resources: Format: "PARQUET" Compression: "PARQUET" OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-cur2" + Name: !Sub '${ResourcePrefix}-cur2' RefreshCadence: Frequency: "SYNCHRONOUS" @@ -559,7 +526,7 @@ Resources: Format: "PARQUET" Compression: "PARQUET" OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-focus" + Name: !Sub '${ResourcePrefix}-focus' RefreshCadence: Frequency: "SYNCHRONOUS" @@ -584,7 +551,7 @@ Resources: Format: "PARQUET" Compression: "PARQUET" OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-focus" + Name: !Sub '${ResourcePrefix}-focus' RefreshCadence: Frequency: "SYNCHRONOUS" @@ -603,8 +570,8 @@ Resources: QueryStatement: !FindInMap [DataExports, COH, DefaultQuery] TableConfigurations: COST_OPTIMIZATION_RECOMMENDATIONS: - FILTER: "{}" - INCLUDE_ALL_RECOMMENDATIONS: "TRUE" + FILTER: '{}' + INCLUDE_ALL_RECOMMENDATIONS: 'TRUE' Description: "Cost Optimization Hub Recommendations export for aggregation in CID" DestinationConfigurations: S3Destination: @@ -616,7 +583,7 @@ Resources: Format: "PARQUET" Compression: "PARQUET" OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-coh" + Name: !Sub '${ResourcePrefix}-coh' RefreshCadence: Frequency: "SYNCHRONOUS" @@ -634,8 +601,8 @@ Resources: QueryStatement: !FindInMap [DataExports, COH, DefaultQuery] TableConfigurations: COST_OPTIMIZATION_RECOMMENDATIONS: - FILTER: "{}" - INCLUDE_ALL_RECOMMENDATIONS: "TRUE" + FILTER: '{}' + INCLUDE_ALL_RECOMMENDATIONS: 'TRUE' Description: "Cost Optimization Hub Recommendations export for aggregation in CID" DestinationConfigurations: S3Destination: @@ -647,18 +614,18 @@ Resources: Format: "PARQUET" Compression: "PARQUET" OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-coh" + Name: !Sub '${ResourcePrefix}-coh' RefreshCadence: Frequency: "SYNCHRONOUS" # COH export requires a service linked role to be created BUT this role can be already created. Thus we need to create it via custom resource LambdaServiceLinkedRoleExecutionRole: - Type: "AWS::IAM::Role" + Type: 'AWS::IAM::Role' Condition: DeployCOHServiceRole Properties: RoleName: !Sub "${AWS::StackName}-LambdaExecutionRole" AssumeRolePolicyDocument: - Version: "2012-10-17" + Version: '2012-10-17' Statement: - Effect: Allow Principal: @@ -668,7 +635,7 @@ Resources: Policies: - PolicyName: !Sub "${AWS::StackName}-LambdaPolicy" PolicyDocument: - Version: "2012-10-17" + Version: '2012-10-17' Statement: - Effect: Allow Action: @@ -683,22 +650,23 @@ Resources: Action: - iam:GetRole - iam:CreateServiceLinkedRole - Resource: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/aws-service-role/bcm-data-exports.amazonaws.com/AWSServiceRoleForBCMDataExports" + Resource: !Sub 'arn:${AWS::Partition}:iam::${AWS::AccountId}:role/aws-service-role/bcm-data-exports.amazonaws.com/AWSServiceRoleForBCMDataExports' - Effect: Allow Action: - cost-optimization-hub:ListEnrollmentStatuses - Resource: "*" # Cannot restrict this + Resource: '*' # Cannot restrict this Metadata: cfn_nag: rules_to_suppress: - - id: "W28" + - id: 'W28' reason: "Need an explicit name for reference" - - id: "W11" + - id: 'W11' reason: "Some COH resources cannot be restricted" + CreateServiceLinkedRoleFunction: - Type: "AWS::Lambda::Function" + Type: 'AWS::Lambda::Function' Condition: DeployCOHServiceRole Properties: FunctionName: !Sub "${ResourcePrefix}-CreateServiceLinkedRoleFunction" @@ -802,21 +770,21 @@ Resources: Metadata: cfn_nag: rules_to_suppress: - - id: "W89" + - id: 'W89' reason: "This Lambda does not require VPC" - - id: "W92" + - id: 'W92' reason: "One Time execution. No need for ReservedConcurrentExecutions" CreateServiceLinkedRoleCustomResource: Condition: DeployCOHServiceRole - Type: "AWS::CloudFormation::CustomResource" + Type: 'AWS::CloudFormation::CustomResource' Properties: ServiceToken: !GetAtt CreateServiceLinkedRoleFunction.Arn ServiceTimeout: "90" - ########################################################################### - # Lambda DataExport Creator: used to create DataExport from outside us-east-1 or cn-northwest-1 - ########################################################################### +########################################################################### +# Lambda DataExport Creator: used to create DataExport from outside us-east-1 or cn-northwest-1 +########################################################################### CidDataExportCreatorLambdaRole: Type: AWS::IAM::Role @@ -835,36 +803,40 @@ Resources: Policies: - PolicyName: "ExecutionDefault" PolicyDocument: - Version: "2012-10-17" + Version: '2012-10-17' Statement: - - Effect: Allow - Action: - - logs:CreateLogStream - - logs:PutLogEvents - - logs:CreateLogGroup - Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator" - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator:*" - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator:*:*" + - Effect: Allow + Action: + - logs:CreateLogStream + - logs:PutLogEvents + - logs:CreateLogGroup + Resource: + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator" + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator:*" + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator:*:*" - PolicyName: "AllowDataExports" PolicyDocument: - Version: "2012-10-17" + Version: '2012-10-17' Statement: - - Effect: Allow - Action: - - bcm-data-exports:CreateExport - - bcm-data-exports:UpdateExport - - bcm-data-exports:DeleteExport - Resource: !Sub "arn:${AWS::Partition}:bcm-data-exports:*:${AWS::AccountId}:*" - - Effect: Allow - Action: - - cur:PutReportDefinition #need this permission for bcm-data-exports to work - Resource: !Sub "arn:${AWS::Partition}:cur:*:${AWS::AccountId}:*" - - Effect: Allow - Action: - - cost-optimization-hub:GetRecommendation #need this permission for bcm-data-exports to work - - cost-optimization-hub:ListRecommendations - Resource: "arn:aws:cost-optimization-hub:*" + - Effect: Allow + Action: + - bcm-data-exports:CreateExport + - bcm-data-exports:UpdateExport + - bcm-data-exports:DeleteExport + Resource: !Sub "arn:${AWS::Partition}:bcm-data-exports:*:${AWS::AccountId}:*" + - Effect: Allow + Action: + - bcm-data-exports:ListExports + Resource: !Sub "arn:${AWS::Partition}:bcm-data-exports:*:${AWS::AccountId}:*" + - Effect: Allow + Action: + - cur:PutReportDefinition #need this permission for bcm-data-exports to work + Resource: !Sub "arn:${AWS::Partition}:cur:*:${AWS::AccountId}:*" + - Effect: Allow + Action: + - cost-optimization-hub:GetRecommendation #need this permission for bcm-data-exports to work + - cost-optimization-hub:ListRecommendations + Resource: "arn:aws:cost-optimization-hub:*" CidDataExportCreatorLambda: Type: AWS::Lambda::Function @@ -897,30 +869,40 @@ Resources: try: export = event['ResourceProperties']['Export'] - export_name = event['ResourceProperties']['Export']['Name'] - + old_export = event.get('OldResourceProperties', {}).get('Export') if event['RequestType'] == 'Create': res = client.create_export(Export=export) print('created:', json.dumps(res)) elif event['RequestType'] == 'Update': - old_export_name = event['OldResourceProperties']['Export']['Name'] - if export["Name"] != old_export_name: + old_export_name = old_export['Name'] + old_export_arn = list(client.get_paginator('list_exports').paginate().search(f"Exports[? ExportName=='{old_export_name}']"))[0]['ExportArn'] + if old_export['Name'] != old_export['Name']: res = client.create_export(Export=export) print('created:', json.dumps(res)) try: - res = client.delete_export(Name=old_export_name) + res = client.delete_export(ExportArn=old_export_arn) print('deleted:', json.dumps(res)) - except: - pass # Do not block deletion + except Exception as exc: + print('Exception on deletion:', str(exc)) + elif export['DataQuery'] != old_export['DataQuery']: # Data export do not support update of queries + print(export['DataQuery'], '!=', old_export['DataQuery']) + res = client.delete_export(ExportArn=old_export_arn) + print('deleted:', json.dumps(res)) + res = client.create_export(Export=export) + print('created:', json.dumps(res)) else: - res = client.update_export(Name=old_export_name, Export=export) + res = client.update_export(ExportArn=old_export_arn, Export=export) print('updated:', json.dumps(res)) elif event['RequestType'] == 'Delete': + export_name = export['Name'] try: - res = client.delete_export(Name=old_export_name) + export_arn = list(client.get_paginator('list_exports').paginate().search(f"Exports[? ExportName=='{export_name}']"))[0]['ExportArn'] + res = client.delete_export(ExportArn=export_arn) print('deleted:', json.dumps(res)) + except client.exceptions.ResourceNotFoundException: + print('Not Found. Nothing to delete') except: - pass # Do not block deletion + print('Exception on deletion:', str(exc)) else: raise Exception("Unknown operation: " + event['RequestType']) @@ -938,9 +920,9 @@ Resources: Metadata: cfn_nag: rules_to_suppress: - - id: "W89" + - id: 'W89' reason: "This Lambda does not require VPC" - - id: "W92" + - id: 'W92' reason: "One Time execution. No need for ReservedConcurrentExecutions" CIDDatabase: @@ -948,14 +930,14 @@ Resources: Condition: DeployAnyTable Properties: DatabaseInput: - Name: !Join ["_", !Split ["-", !Sub "${ResourcePrefix}_data_export"]] # replace '-' to '_' + Name: !Join [ '_', !Split [ '-', !Sub '${ResourcePrefix}_data_export' ] ] # replace '-' to '_' CatalogId: !Sub "${AWS::AccountId}" - ########################################################################### - # CUR2 - ########################################################################### +########################################################################### +# CUR2 +########################################################################### - CURTable: # Initial creation of table. it will be updated by crawler later + CURTable: # Initial creation of table. it will be updated by crawler later Type: AWS::Glue::Table Condition: DeployCUR2Table Properties: @@ -979,108 +961,88 @@ Resources: OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat SerdeInfo: Parameters: - serialization.format: "1" + serialization.format: '1' SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe StoredAsSubDirectories: false Columns: # All fields required for CID - - { "Name": "bill_bill_type", "Type": "string" } - - { "Name": "bill_billing_entity", "Type": "string" } - - { "Name": "bill_billing_period_end_date", "Type": "timestamp" } - - { "Name": "bill_billing_period_start_date", "Type": "timestamp" } - - { "Name": "bill_invoice_id", "Type": "string" } - - { "Name": "bill_payer_account_id", "Type": "string" } - - { "Name": "bill_payer_account_name", "Type": "string" } - - { "Name": "cost_category", "Type": "map" } - - { "Name": "discount", "Type": "map" } - - { "Name": "identity_line_item_id", "Type": "string" } - - { "Name": "identity_time_interval", "Type": "string" } - - { "Name": "line_item_availability_zone", "Type": "string" } - - { "Name": "line_item_legal_entity", "Type": "string" } - - { "Name": "line_item_line_item_description", "Type": "string" } - - { "Name": "line_item_line_item_type", "Type": "string" } - - { "Name": "line_item_operation", "Type": "string" } - - { "Name": "line_item_product_code", "Type": "string" } - - { "Name": "line_item_resource_id", "Type": "string" } - - { "Name": "line_item_unblended_cost", "Type": "double" } - - { "Name": "line_item_usage_account_id", "Type": "string" } - - { "Name": "line_item_usage_account_name", "Type": "string" } - - { "Name": "line_item_usage_amount", "Type": "double" } - - { "Name": "line_item_usage_end_date", "Type": "timestamp" } - - { "Name": "line_item_usage_start_date", "Type": "timestamp" } - - { "Name": "line_item_usage_type", "Type": "string" } - - { "Name": "pricing_lease_contract_length", "Type": "string" } - - { "Name": "pricing_offering_class", "Type": "string" } - - { "Name": "pricing_public_on_demand_cost", "Type": "double" } - - { "Name": "pricing_purchase_option", "Type": "string" } - - { "Name": "pricing_term", "Type": "string" } - - { "Name": "pricing_unit", "Type": "string" } - - { "Name": "product", "Type": "map" } - - { "Name": "product_from_location", "Type": "string" } - - { "Name": "product_instance_type", "Type": "string" } - - { "Name": "product_product_family", "Type": "string" } - - { "Name": "product_servicecode", "Type": "string" } - - { "Name": "product_to_location", "Type": "string" } - - { - "Name": "reservation_amortized_upfront_fee_for_billing_period", - "Type": "double", - } - - { "Name": "reservation_effective_cost", "Type": "double" } - - { "Name": "reservation_end_time", "Type": "string" } - - { "Name": "reservation_reservation_a_r_n", "Type": "string" } - - { "Name": "reservation_start_time", "Type": "string" } - - { - "Name": "reservation_unused_amortized_upfront_fee_for_billing_period", - "Type": "double", - } - - { "Name": "reservation_unused_recurring_fee", "Type": "double" } - - { "Name": "resource_tags", "Type": "map" } - - { - "Name": "savings_plan_amortized_upfront_commitment_for_billing_period", - "Type": "double", - } - - { "Name": "savings_plan_end_time", "Type": "string" } - - { "Name": "savings_plan_offering_type", "Type": "string" } - - { "Name": "savings_plan_payment_option", "Type": "string" } - - { "Name": "savings_plan_purchase_term", "Type": "string" } - - { "Name": "savings_plan_savings_plan_a_r_n", "Type": "string" } - - { - "Name": "savings_plan_savings_plan_effective_cost", - "Type": "double", - } - - { "Name": "savings_plan_start_time", "Type": "string" } - - { - "Name": "savings_plan_total_commitment_to_date", - "Type": "double", - } - - { "Name": "savings_plan_used_commitment", "Type": "double" } - - { "Name": "split_line_item_parent_resource_id", "Type": "string" } - - { "Name": "split_line_item_reserved_usage", "Type": "double" } - - { "Name": "split_line_item_actual_usage", "Type": "double" } - - { "Name": "split_line_item_split_usage", "Type": "double" } - - { "Name": "split_line_item_split_usage_ratio", "Type": "double" } - - { "Name": "split_line_item_split_cost", "Type": "double" } - - { "Name": "split_line_item_unused_cost", "Type": "double" } - - { "Name": "split_line_item_net_split_cost", "Type": "double" } - - { "Name": "split_line_item_net_unused_cost", "Type": "double" } - - { - "Name": "split_line_item_public_on_demand_split_cost", - "Type": "double", - } - - { - "Name": "split_line_item_public_on_demand_unused_cost", - "Type": "double", - } + - {"Name": "bill_bill_type", "Type": "string" } + - {"Name": "bill_billing_entity", "Type": "string" } + - {"Name": "bill_billing_period_end_date", "Type": "timestamp" } + - {"Name": "bill_billing_period_start_date", "Type": "timestamp" } + - {"Name": "bill_invoice_id", "Type": "string" } + - {"Name": "bill_payer_account_id", "Type": "string" } + - {"Name": "bill_payer_account_name", "Type": "string" } + - {"Name": "cost_category", "Type": "map" } + - {"Name": "discount", "Type": "map" } + - {"Name": "identity_line_item_id", "Type": "string" } + - {"Name": "identity_time_interval", "Type": "string" } + - {"Name": "line_item_availability_zone", "Type": "string" } + - {"Name": "line_item_legal_entity", "Type": "string" } + - {"Name": "line_item_line_item_description", "Type": "string" } + - {"Name": "line_item_line_item_type", "Type": "string" } + - {"Name": "line_item_operation", "Type": "string" } + - {"Name": "line_item_product_code", "Type": "string" } + - {"Name": "line_item_resource_id", "Type": "string" } + - {"Name": "line_item_unblended_cost", "Type": "double" } + - {"Name": "line_item_usage_account_id", "Type": "string" } + - {"Name": "line_item_usage_account_name", "Type": "string" } + - {"Name": "line_item_usage_amount", "Type": "double" } + - {"Name": "line_item_usage_end_date", "Type": "timestamp" } + - {"Name": "line_item_usage_start_date", "Type": "timestamp" } + - {"Name": "line_item_usage_type", "Type": "string" } + - {"Name": "pricing_lease_contract_length", "Type": "string" } + - {"Name": "pricing_offering_class", "Type": "string" } + - {"Name": "pricing_public_on_demand_cost", "Type": "double" } + - {"Name": "pricing_purchase_option", "Type": "string" } + - {"Name": "pricing_term", "Type": "string" } + - {"Name": "pricing_unit", "Type": "string" } + - {"Name": "product", "Type": "map" } + - {"Name": "product_from_location", "Type": "string" } + - {"Name": "product_instance_type", "Type": "string" } + - {"Name": "product_product_family", "Type": "string" } + - {"Name": "product_servicecode", "Type": "string" } + - {"Name": "product_to_location", "Type": "string" } + - {"Name": "reservation_amortized_upfront_fee_for_billing_period", "Type": "double" } + - {"Name": "reservation_effective_cost", "Type": "double" } + - {"Name": "reservation_end_time", "Type": "string" } + - {"Name": "reservation_reservation_a_r_n", "Type": "string" } + - {"Name": "reservation_start_time", "Type": "string" } + - {"Name": "reservation_unused_amortized_upfront_fee_for_billing_period", "Type": "double" } + - {"Name": "reservation_unused_recurring_fee", "Type": "double" } + - {"Name": "resource_tags", "Type": "map" } + - {"Name": "savings_plan_amortized_upfront_commitment_for_billing_period", "Type": "double" } + - {"Name": "savings_plan_end_time", "Type": "string" } + - {"Name": "savings_plan_offering_type", "Type": "string" } + - {"Name": "savings_plan_payment_option", "Type": "string" } + - {"Name": "savings_plan_purchase_term", "Type": "string" } + - {"Name": "savings_plan_savings_plan_a_r_n", "Type": "string" } + - {"Name": "savings_plan_savings_plan_effective_cost", "Type": "double" } + - {"Name": "savings_plan_start_time", "Type": "string" } + - {"Name": "savings_plan_total_commitment_to_date", "Type": "double" } + - {"Name": "savings_plan_used_commitment", "Type": "double" } + - {"Name": "split_line_item_parent_resource_id", "Type": "string" } + - {"Name": "split_line_item_reserved_usage", "Type": "double" } + - {"Name": "split_line_item_actual_usage", "Type": "double" } + - {"Name": "split_line_item_split_usage", "Type": "double" } + - {"Name": "split_line_item_split_usage_ratio", "Type": "double" } + - {"Name": "split_line_item_split_cost", "Type": "double" } + - {"Name": "split_line_item_unused_cost", "Type": "double" } + - {"Name": "split_line_item_net_split_cost", "Type": "double" } + - {"Name": "split_line_item_net_unused_cost", "Type": "double" } + - {"Name": "split_line_item_public_on_demand_split_cost", "Type": "double" } + - {"Name": "split_line_item_public_on_demand_unused_cost", "Type": "double" } PartitionKeys: - - { "Name": "source_account_id", "Type": "string" } - - { "Name": "report_name", "Type": "string" } - - { "Name": "data", "Type": "string" } - - { "Name": "billing_period", "Type": "string" } + - {"Name": "source_account_id", "Type": "string" } + - {"Name": "report_name", "Type": "string" } + - {"Name": "data", "Type": "string" } + - {"Name": "billing_period", "Type": "string" } + CURCrawler: Type: AWS::Glue::Crawler Condition: DeployCUR2Table Properties: - Name: !Sub "${ResourcePrefix}-DataExportCUR2Crawler" + Name: !Sub "${ResourcePrefix}-DataExportCUR2Crawler" Description: A recurring crawler that keeps your CUR table in Athena up-to-date. Role: !GetAtt CidDataExportCrawlerRole.Arn DatabaseName: !Ref CIDDatabase @@ -1088,15 +1050,15 @@ Resources: S3Targets: - Path: !Sub "s3://${DestinationS3}/cur2/" Exclusions: - - "**.json" - - "**.yml" - - "**.sql" - - "**.csv" - - "**.csv.metadata" - - "**.gz" - - "**.zip" - - "**/cost_and_usage_data_status/*" - - "aws-programmatic-access-test-object" + - '**.json' + - '**.yml' + - '**.sql' + - '**.csv' + - '**.csv.metadata' + - '**.gz' + - '**.zip' + - '**/cost_and_usage_data_status/*' + - 'aws-programmatic-access-test-object' SchemaChangePolicy: DeleteBehavior: LOG RecrawlPolicy: @@ -1116,11 +1078,11 @@ Resources: } } - ########################################################################### - # FOCUS - ########################################################################### +########################################################################### +# FOCUS +########################################################################### - FOCUSTable: # Initial creation of table. it will be updated by crawler later + FOCUSTable: # Initial creation of table. it will be updated by crawler later Type: AWS::Glue::Table Condition: DeployFOCUSTable Properties: @@ -1144,64 +1106,65 @@ Resources: OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat SerdeInfo: Parameters: - serialization.format: "1" + serialization.format: '1' SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe StoredAsSubDirectories: false Columns: - - { "Name": "availabilityzone", "Type": "string" } - - { "Name": "billedcost", "Type": "double" } - - { "Name": "billingaccountid", "Type": "string" } - - { "Name": "billingaccountname", "Type": "string" } - - { "Name": "billingcurrency", "Type": "string" } - - { "Name": "billingperiodend", "Type": "timestamp" } - - { "Name": "billingperiodstart", "Type": "timestamp" } - - { "Name": "chargecategory", "Type": "string" } - - { "Name": "chargeclass", "Type": "string" } - - { "Name": "chargedescription", "Type": "string" } - - { "Name": "chargefrequency", "Type": "string" } - - { "Name": "chargeperiodend", "Type": "timestamp" } - - { "Name": "chargeperiodstart", "Type": "timestamp" } - - { "Name": "commitmentdiscountcategory", "Type": "string" } - - { "Name": "commitmentdiscountid", "Type": "string" } - - { "Name": "commitmentdiscountname", "Type": "string" } - - { "Name": "commitmentdiscounttype", "Type": "string" } - - { "Name": "commitmentdiscountstatus", "Type": "string" } - - { "Name": "consumedquantity", "Type": "double" } - - { "Name": "consumedunit", "Type": "string" } - - { "Name": "contractedcost", "Type": "double" } - - { "Name": "contractedunitprice", "Type": "double" } - - { "Name": "effectivecost", "Type": "double" } - - { "Name": "invoiceissuername", "Type": "string" } - - { "Name": "listcost", "Type": "double" } - - { "Name": "listunitprice", "Type": "double" } - - { "Name": "pricingcategory", "Type": "string" } - - { "Name": "pricingquantity", "Type": "double" } - - { "Name": "pricingunit", "Type": "string" } - - { "Name": "providername", "Type": "string" } - - { "Name": "publishername", "Type": "string" } - - { "Name": "regionid", "Type": "string" } - - { "Name": "regionname", "Type": "string" } - - { "Name": "resourceid", "Type": "string" } - - { "Name": "resourcename", "Type": "string" } - - { "Name": "resourcetype", "Type": "string" } - - { "Name": "servicecategory", "Type": "string" } - - { "Name": "servicename", "Type": "string" } - - { "Name": "skuid", "Type": "string" } - - { "Name": "skupriceid", "Type": "string" } - - { "Name": "subaccountid", "Type": "string" } - - { "Name": "subaccountname", "Type": "string" } - - { "Name": "tags", "Type": "map" } + - {"Name": "availabilityzone", "Type": "string" } + - {"Name": "billedcost", "Type": "double" } + - {"Name": "billingaccountid", "Type": "string" } + - {"Name": "billingaccountname", "Type": "string" } + - {"Name": "billingcurrency", "Type": "string" } + - {"Name": "billingperiodend", "Type": "timestamp" } + - {"Name": "billingperiodstart", "Type": "timestamp" } + - {"Name": "chargecategory", "Type": "string" } + - {"Name": "chargeclass", "Type": "string" } + - {"Name": "chargedescription", "Type": "string" } + - {"Name": "chargefrequency", "Type": "string" } + - {"Name": "chargeperiodend", "Type": "timestamp" } + - {"Name": "chargeperiodstart", "Type": "timestamp" } + - {"Name": "commitmentdiscountcategory", "Type": "string" } + - {"Name": "commitmentdiscountid", "Type": "string" } + - {"Name": "commitmentdiscountname", "Type": "string" } + - {"Name": "commitmentdiscounttype", "Type": "string" } + - {"Name": "commitmentdiscountstatus", "Type": "string" } + - {"Name": "consumedquantity", "Type": "double" } + - {"Name": "consumedunit", "Type": "string" } + - {"Name": "contractedcost", "Type": "double" } + - {"Name": "contractedunitprice", "Type": "double" } + - {"Name": "effectivecost", "Type": "double" } + - {"Name": "invoiceissuername", "Type": "string" } + - {"Name": "listcost", "Type": "double" } + - {"Name": "listunitprice", "Type": "double" } + - {"Name": "pricingcategory", "Type": "string" } + - {"Name": "pricingquantity", "Type": "double" } + - {"Name": "pricingunit", "Type": "string" } + - {"Name": "providername", "Type": "string" } + - {"Name": "publishername", "Type": "string" } + - {"Name": "regionid", "Type": "string" } + - {"Name": "regionname", "Type": "string" } + - {"Name": "resourceid", "Type": "string" } + - {"Name": "resourcename", "Type": "string" } + - {"Name": "resourcetype", "Type": "string" } + - {"Name": "servicecategory", "Type": "string" } + - {"Name": "servicename", "Type": "string" } + - {"Name": "skuid", "Type": "string" } + - {"Name": "skupriceid", "Type": "string" } + - {"Name": "subaccountid", "Type": "string" } + - {"Name": "subaccountname", "Type": "string" } + - {"Name": "tags", "Type": "map" } PartitionKeys: - - { "Name": "source_account_id", "Type": "string" } - - { "Name": "report_name", "Type": "string" } - - { "Name": "data", "Type": "string" } - - { "Name": "billing_period", "Type": "string" } + - {"Name": "source_account_id", "Type": "string" } + - {"Name": "report_name", "Type": "string" } + - {"Name": "data", "Type": "string" } + - {"Name": "billing_period", "Type": "string" } + FOCUSCrawler: Type: AWS::Glue::Crawler Condition: DeployFOCUSTable Properties: - Name: !Sub "${ResourcePrefix}-DataExportFOCUSCrawler" + Name: !Sub "${ResourcePrefix}-DataExportFOCUSCrawler" Description: A recurring crawler that keeps your FOCUS table in Athena up-to-date. Role: !GetAtt CidDataExportCrawlerRole.Arn DatabaseName: !Ref CIDDatabase @@ -1209,15 +1172,15 @@ Resources: S3Targets: - Path: !Sub "s3://${DestinationS3}/focus/" Exclusions: - - "**.json" - - "**.yml" - - "**.sql" - - "**.csv" - - "**.csv.metadata" - - "**.gz" - - "**.zip" - - "**/cost_and_usage_data_status/*" - - "aws-programmatic-access-test-object" + - '**.json' + - '**.yml' + - '**.sql' + - '**.csv' + - '**.csv.metadata' + - '**.gz' + - '**.zip' + - '**/cost_and_usage_data_status/*' + - 'aws-programmatic-access-test-object' SchemaChangePolicy: DeleteBehavior: LOG RecrawlPolicy: @@ -1237,11 +1200,11 @@ Resources: } } - ########################################################################### - # COH - ########################################################################### +########################################################################### +# COH +########################################################################### - COHTable: # Initial creation of table. it will be updated by crawler later + COHTable: # Initial creation of table. it will be updated by crawler later Type: AWS::Glue::Table Condition: DeployCOHTable Properties: @@ -1265,67 +1228,47 @@ Resources: OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat SerdeInfo: Parameters: - serialization.format: "1" + serialization.format: '1' SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe StoredAsSubDirectories: false Columns: # All fields required for CID - - { "Name": "account_id", "Type": "string" } - - { "Name": "action_type", "Type": "string" } - - { "Name": "currency_code", "Type": "string" } - - { "Name": "current_resource_details", "Type": "string" } - - { "Name": "current_resource_summary", "Type": "string" } - - { "Name": "current_resource_type", "Type": "string" } - - { - "Name": "estimated_monthly_cost_after_discount", - "Type": "double", - } - - { - "Name": "estimated_monthly_cost_before_discount", - "Type": "double", - } - - { - "Name": "estimated_monthly_savings_after_discount", - "Type": "double", - } - - { - "Name": "estimated_monthly_savings_before_discount", - "Type": "double", - } - - { - "Name": "estimated_savings_percentage_after_discount", - "Type": "double", - } - - { - "Name": "estimated_savings_percentage_before_discount", - "Type": "double", - } - - { "Name": "implementation_effort", "Type": "string" } - - { "Name": "last_refresh_timestamp", "Type": "string" } - - { "Name": "recommendation_id", "Type": "string" } - - { - "Name": "recommendation_lookback_period_in_days", - "Type": "int", - } - - { "Name": "recommendation_source", "Type": "string" } - - { "Name": "recommended_resource_details", "Type": "string" } - - { "Name": "recommended_resource_summary", "Type": "string" } - - { "Name": "recommended_resource_type", "Type": "string" } - - { "Name": "region", "Type": "string" } - - { "Name": "resource_arn", "Type": "string" } - - { "Name": "restart_needed", "Type": "boolean" } - - { "Name": "rollback_possible", "Type": "boolean" } - - { "Name": "tags", "Type": "map" } + - {"Name": "account_id", "Type": "string"} + - {"Name": "action_type", "Type": "string"} + - {"Name": "currency_code", "Type": "string"} + - {"Name": "current_resource_details", "Type": "string"} + - {"Name": "current_resource_summary", "Type": "string"} + - {"Name": "current_resource_type", "Type": "string"} + - {"Name": "estimated_monthly_cost_after_discount", "Type": "double"} + - {"Name": "estimated_monthly_cost_before_discount", "Type": "double"} + - {"Name": "estimated_monthly_savings_after_discount", "Type": "double"} + - {"Name": "estimated_monthly_savings_before_discount", "Type": "double"} + - {"Name": "estimated_savings_percentage_after_discount", "Type": "double"} + - {"Name": "estimated_savings_percentage_before_discount", "Type": "double"} + - {"Name": "implementation_effort", "Type": "string"} + - {"Name": "last_refresh_timestamp", "Type": "string"} + - {"Name": "recommendation_id", "Type": "string"} + - {"Name": "recommendation_lookback_period_in_days", "Type": "int"} + - {"Name": "recommendation_source", "Type": "string"} + - {"Name": "recommended_resource_details", "Type": "string"} + - {"Name": "recommended_resource_summary", "Type": "string"} + - {"Name": "recommended_resource_type", "Type": "string"} + - {"Name": "region", "Type": "string"} + - {"Name": "resource_arn", "Type": "string"} + - {"Name": "restart_needed", "Type": "boolean"} + - {"Name": "rollback_possible", "Type": "boolean"} + - {"Name": "tags", "Type": "map"} PartitionKeys: - - { "Name": "source_account_id", "Type": "string" } - - { "Name": "report_name", "Type": "string" } - - { "Name": "data", "Type": "string" } - - { "Name": "date", "Type": "string" } + - {"Name": "source_account_id", "Type": "string" } + - {"Name": "report_name", "Type": "string" } + - {"Name": "data", "Type": "string" } + - {"Name": "date", "Type": "string" } + COHCrawler: Type: AWS::Glue::Crawler Condition: DeployCOHTable Properties: - Name: !Sub "${ResourcePrefix}-DataExportCOHCrawler" + Name: !Sub "${ResourcePrefix}-DataExportCOHCrawler" Description: A recurring crawler that keeps your COH table in Athena up-to-date. Role: !GetAtt CidDataExportCrawlerRole.Arn DatabaseName: !Ref CIDDatabase @@ -1333,15 +1276,15 @@ Resources: S3Targets: - Path: !Sub "s3://${DestinationS3}/coh/" Exclusions: - - "**.json" - - "**.yml" - - "**.sql" - - "**.csv" - - "**.csv.metadata" - - "**.gz" - - "**.zip" - - "**/cost_and_usage_data_status/*" - - "aws-programmatic-access-test-object" + - '**.json' + - '**.yml' + - '**.sql' + - '**.csv' + - '**.csv.metadata' + - '**.gz' + - '**.zip' + - '**/cost_and_usage_data_status/*' + - 'aws-programmatic-access-test-object' SchemaChangePolicy: DeleteBehavior: LOG RecrawlPolicy: @@ -1361,9 +1304,70 @@ Resources: } } - ########################################################################### - # Generic Resources - ########################################################################### + + LakeFormationTagsForDatabase: + Type: AWS::LakeFormation::TagAssociation + Condition: NeedLakeFormationEnabledDB + Properties: + Resource: + Database: + CatalogId: !Ref "AWS::AccountId" + Name: !Ref CIDDatabase + LFTags: + - TagKey: !ImportValue cid-LakeFormation-TagKey # you need to install cid-lakeformation-prerequisite.yaml + TagValues: + - !ImportValue cid-LakeFormation-TagValue + CatalogId: !Ref "AWS::AccountId" + + LakeFormationTagsForCUR2: + Type: AWS::LakeFormation::TagAssociation + Condition: NeedLakeFormationEnabledCUR2 + Properties: + Resource: + Table: + CatalogId: !Ref "AWS::AccountId" + DatabaseName: !Ref CIDDatabase + Name: !Ref CURTable + LFTags: + - TagKey: !ImportValue cid-LakeFormation-TagKey # you need to install cid-lakeformation-prerequisite.yaml + TagValues: + - !ImportValue cid-LakeFormation-TagValue + CatalogId: !Ref "AWS::AccountId" + + LakeFormationTagsForCOH: + Type: AWS::LakeFormation::TagAssociation + Condition: NeedLakeFormationEnabledCOH + Properties: + Resource: + Table: + CatalogId: !Ref "AWS::AccountId" + DatabaseName: !Ref CIDDatabase + Name: !Ref COHTable + LFTags: + - TagKey: !ImportValue cid-LakeFormation-TagKey # you need to install cid-lakeformation-prerequisite.yaml + TagValues: + - !ImportValue cid-LakeFormation-TagValue + CatalogId: !Ref "AWS::AccountId" + + LakeFormationTagsForFOCUS: + Type: AWS::LakeFormation::TagAssociation + Condition: NeedLakeFormationEnabledFOCUS + Properties: + Resource: + Table: + CatalogId: !Ref "AWS::AccountId" + DatabaseName: !Ref CIDDatabase + Name: !Ref FOCUSTable + LFTags: + - TagKey: !ImportValue cid-LakeFormation-TagKey # you need to install cid-lakeformation-prerequisite.yaml + TagValues: + - !ImportValue cid-LakeFormation-TagValue + CatalogId: !Ref "AWS::AccountId" + + +########################################################################### +# Generic Resources +########################################################################### CidDataExportCrawlerRole: Type: AWS::IAM::Role @@ -1377,7 +1381,7 @@ Resources: Service: - glue.amazonaws.com Action: - - "sts:AssumeRole" + - 'sts:AssumeRole' Path: !Ref RolePath Policies: - PolicyName: CrawlerPolicy @@ -1386,12 +1390,12 @@ Resources: Statement: - Effect: Allow Action: - - "s3:GetObject" - Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" + - 's3:GetObject' + Resource: !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}/*' - Effect: Allow Action: - - "s3:ListBucket" - Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" + - 's3:ListBucket' + Resource: !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}' - Effect: Allow Action: - glue:GetDatabase @@ -1419,16 +1423,16 @@ Resources: - logs:CreateLogGroup - logs:CreateLogStream Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws-glue/crawlers:*" + - !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws-glue/crawlers:*' - Effect: Allow Action: - logs:PutLogEvents Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws-glue/crawlers:log-stream:*" + - !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws-glue/crawlers:log-stream:*' - ########################################################################### - # Analytics: used by CID team to track adoption, by retrieving AWS AccountId - ########################################################################### +########################################################################### +# Analytics: used by CID team to track adoption, by retrieving AWS AccountId +########################################################################### CidLambdaAnalyticsRole: #Execution role for the custom resource CidLambdaAnalyticsExecutor Type: AWS::IAM::Role @@ -1447,17 +1451,17 @@ Resources: Policies: - PolicyName: "ExecutionDefault" PolicyDocument: - Version: "2012-10-17" + Version: '2012-10-17' Statement: - - Effect: Allow - Action: - - logs:CreateLogStream - - logs:PutLogEvents - - logs:CreateLogGroup - Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports" - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports:*" - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports:*:*" + - Effect: Allow + Action: + - logs:CreateLogStream + - logs:PutLogEvents + - logs:CreateLogGroup + Resource: + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports" + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports:*" + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports:*:*" CidLambdaAnalytics: Type: AWS::Lambda::Function @@ -1524,9 +1528,9 @@ Resources: Metadata: cfn_nag: rules_to_suppress: - - id: "W89" + - id: 'W89' reason: "This Lambda does not require VPC" - - id: "W92" + - id: 'W92' reason: "One Time execution. No need for ReservedConcurrentExecutions" CidLambdaAnalyticsExecutorForDataExports: @@ -1534,21 +1538,17 @@ Resources: Properties: ServiceToken: !GetAtt CidLambdaAnalytics.Arn DeploymentType: - - !If [ - IsDestinationAccount, - "cid-dataexport-destination", - !Ref "AWS::NoValue", - ] - - !If [IsSourceAccount, "cid-dataexport-source", !Ref "AWS::NoValue"] + - !If [IsDestinationAccount, "cid-dataexport-destination", !Ref 'AWS::NoValue'] + - !If [IsSourceAccount, "cid-dataexport-source", !Ref 'AWS::NoValue'] DataExportsReadAccess: Type: AWS::IAM::ManagedPolicy Condition: DeployAnyTable Properties: ManagedPolicyName: !Sub ${ResourcePrefix}DataExportsReadAccess - Description: "Policy for QuickSight to allow DataExports access" + Description: 'Policy for QuickSight to allow DataExports access' PolicyDocument: - Version: "2012-10-17" + Version: '2012-10-17' Statement: - Sid: AllowGlue Effect: Allow @@ -1562,20 +1562,14 @@ Resources: Resource: - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog - Fn::Join: - - "" - - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/ - - !Join [ - "_", - !Split ["-", !Sub "${ResourcePrefix}_data_export"], - ] # replace '-' to '_' - - "/*" + - '' + - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/ + - !Join [ '_', !Split [ '-', !Sub '${ResourcePrefix}_data_export' ] ] # replace '-' to '_' + - '/*' - Fn::Join: - - "" - - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/ - - !Join [ - "_", - !Split ["-", !Sub "${ResourcePrefix}_data_export"], - ] # replace '-' to '_' + - '' + - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/ + - !Join [ '_', !Split [ '-', !Sub '${ResourcePrefix}_data_export' ] ] # replace '-' to '_' - Sid: AllowListBucket Effect: Allow Action: s3:ListBucket @@ -1591,24 +1585,26 @@ Resources: Metadata: cfn_nag: rules_to_suppress: - - id: "W28" + - id: 'W28' reason: "Need an explicit name for reference" Outputs: AggregateBucketName: + Condition: IsDestinationAccount Description: Bucket with aggregate Data Exports Value: !Sub ${ResourcePrefix}-${DestinationAccountId}-data-exports - Export: { Name: "cid-DataExports-Bucket" } + Export: { Name: 'cid-DataExports-Bucket'} Database: + Condition: DeployAnyTable Description: Database for Data Exports - Value: !Join ["_", !Split ["-", !Sub "${ResourcePrefix}_data_export"]] # replace '-' to '_' - Export: { Name: "cid-DataExports-Database" } + Value: !Join [ '_', !Split [ '-', !Sub '${ResourcePrefix}_data_export' ] ] # replace '-' to '_' + Export: { Name: 'cid-DataExports-Database'} LocalAccountBucket: - Condition: DeployDataExport + Condition: DeploySourceS3 Description: Local Bucket Name which replicate objects to centralized bucket Value: !Sub ${ResourcePrefix}-${AWS::AccountId}-data-local ReadAccessPolicyARN: Condition: DeployAnyTable Description: Policy to allow read access DataExports in S3 and Athena. Attach it to QuickSight role. - Value: !Ref DataExportsReadAccess - Export: { Name: "cid-DataExports-ReadAccessPolicyARN" } + Value: !Ref DataExportsReadAccess + Export: { Name: 'cid-DataExports-ReadAccessPolicyARN'} diff --git a/examples/basic/README.md b/examples/basic/README.md index ea159e2..af71ac4 100644 --- a/examples/basic/README.md +++ b/examples/basic/README.md @@ -7,10 +7,9 @@ No providers. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [quicksights\_username](#input\_quicksights\_username) | The username to use for QuickSight | `string` | `"admin"` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | 

{
  "Environment": "Production"
}
| no | ## Outputs No outputs. - \ No newline at end of file + diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 0c514f5..f776253 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -18,10 +18,13 @@ module "destination" { dashboards_bucket_name = local.dashboard_bucket_name enable_sso = true payer_accounts = ["1234343434"] - quicksights_username = var.quicksights_username saml_metadata = file("${path.module}/assets/saml-metadata.xml") tags = var.tags + quicksight_admin_email = "quicksight-admin@example.com" + quicksight_admin_username = "admin" + quicksight_dashboard_owner = "admin" + providers = { aws = aws.cost_analysis aws.us_east_1 = aws.cost_analysis_us_east_1 diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf index 6800124..3c5fe5a 100644 --- a/examples/basic/variables.tf +++ b/examples/basic/variables.tf @@ -6,9 +6,3 @@ variable "tags" { Environment = "Production" } } - -variable "quicksights_username" { - description = "The username to use for QuickSight" - type = string - default = "admin" -} diff --git a/modules/destination/README.md b/modules/destination/README.md index dc10aae..bc22a17 100644 --- a/modules/destination/README.md +++ b/modules/destination/README.md @@ -11,6 +11,7 @@ |------|-------------|------|---------|:--------:| | [cloudformation\_bucket\_name](#input\_cloudformation\_bucket\_name) | The name of the bucket to store the CloudFormation | `string` | n/a | yes | | [dashboards\_bucket\_name](#input\_dashboards\_bucket\_name) | The name of the bucket to store the dashboards configurations | `string` | n/a | yes | +| [quicksight\_admin\_email](#input\_quicksight\_admin\_email) | The email address for the QuickSight admin user. Required if var.create\_quicksight\_admin\_user is true | `string` | n/a | yes | | [tags](#input\_tags) | Tags to apply to all resources | `map(string)` | n/a | yes | | [enable\_backup\_module](#input\_enable\_backup\_module) | Indicates if the Backup module should be enabled | `bool` | `true` | no | | [enable\_budgets\_module](#input\_enable\_budgets\_module) | Indicates if the Budget module should be enabled | `bool` | `true` | no | @@ -28,7 +29,7 @@ | [enable\_org\_data\_module](#input\_enable\_org\_data\_module) | Indicates if the Organization Data module should be enabled | `bool` | `true` | no | | [enable\_prerequisites\_quicksight](#input\_enable\_prerequisites\_quicksight) | Indicates if the prerequisites for QuickSight should be enabled | `bool` | `true` | no | | [enable\_prerequisites\_quicksight\_permissions](#input\_enable\_prerequisites\_quicksight\_permissions) | Indicates if the prerequisites for QuickSight permissions should be enabled | `bool` | `true` | no | -| [enable\_quicksight\_admin](#input\_enable\_quicksight\_admin) | Enable the creation of an admin user (var.quicksights\_username) in QuickSight | `bool` | `true` | no | +| [enable\_quicksight\_admin](#input\_enable\_quicksight\_admin) | Enable the creation of an admin user (var.quicksight\_dashboard\_owner) in QuickSight | `bool` | `true` | no | | [enable\_quicksight\_subscription](#input\_enable\_quicksight\_subscription) | Enable QuickSight subscription | `bool` | `false` | no | | [enable\_rds\_utilization\_module](#input\_enable\_rds\_utilization\_module) | Indicates if the RDS Utilization module should be enabled | `bool` | `true` | no | | [enable\_rightsizing\_module](#input\_enable\_rightsizing\_module) | Indicates if the Rightsizing module should be enabled | `bool` | `true` | no | @@ -38,17 +39,16 @@ | [enable\_tao\_module](#input\_enable\_tao\_module) | Indicates if the TAO module should be enabled | `bool` | `true` | no | | [enable\_transit\_gateway\_module](#input\_enable\_transit\_gateway\_module) | Indicates if the Transit Gateway module should be enabled | `bool` | `true` | no | | [payer\_accounts](#input\_payer\_accounts) | List of additional payer accounts to be included in the collectors module | `list(string)` | `[]` | no | -| [quicksight\_admin\_email](#input\_quicksight\_admin\_email) | The email address for the QuickSight admin user | `string` | `null` | no | | [quicksight\_admin\_username](#input\_quicksight\_admin\_username) | The username for the QuickSight admin user | `string` | `"admin"` | no | +| [quicksight\_dashboard\_owner](#input\_quicksight\_dashboard\_owner) | The username for the QuickSight user who will own the dashboards. This user needs to exist. By default, it will be the admin user which is created by the module. | `string` | `"admin"` | no | | [quicksight\_groups](#input\_quicksight\_groups) | Map of groups with user membership to be added to QuickSight | 
map(object({
    description = optional(string)
    namespace   = optional(string)
    members     = optional(list(string), [])
  }))
| `{}` | no | | [quicksight\_subscription\_account\_name](#input\_quicksight\_subscription\_account\_name) | The account name for the QuickSight quicksight\_subscription edition | `string` | `null` | no | | [quicksight\_subscription\_authentication\_method](#input\_quicksight\_subscription\_authentication\_method) | The identity for the QuickSight quicksight\_subscription edition | `string` | `"IAM_AND_QUICKSIGHT"` | no | | [quicksight\_subscription\_edition](#input\_quicksight\_subscription\_edition) | The edition for the QuickSight quicksight\_subscription | `string` | `"ENTERPRISE"` | no | | [quicksight\_subscription\_email](#input\_quicksight\_subscription\_email) | The email address for the QuickSight quicksight\_subscription edition | `string` | `null` | no | -| [quicksight\_users](#input\_quicksight\_users) | Map of user accounts to be registered in QuickSight | 
map(object({
    identity_type = optional(string, "IAM")
    namespace     = optional(string, "default")
    role          = optional(string, "READER")
  }))
| `{}` | no | -| [quicksights\_username](#input\_quicksights\_username) | The username for the QuickSight user | `string` | `"admin"` | no | +| [quicksight\_users](#input\_quicksight\_users) | Map of user accounts to be registered in QuickSight | 
map(object({
    identity_type = string
    namespace     = optional(string, "default")
    role          = optional(string, "READER")
  }))
| `{}` | no | | [saml\_iam\_role\_name](#input\_saml\_iam\_role\_name) | Name of the role all authentication users are initially given | `string` | `"aws-cudos-sso"` | no | -| [saml\_metadata](#input\_saml\_metadata) | The configuration for the SAML identity provider | `string` | `null` | no | +| [saml\_metadata](#input\_saml\_metadata) | The configuration for the SAML identity provider | `string` | `""` | no | | [saml\_provider\_name](#input\_saml\_provider\_name) | The name of the SAML provider | `string` | `"aws-cudos-sso"` | no | | [stack\_name\_cloud\_intelligence](#input\_stack\_name\_cloud\_intelligence) | The name of the CloudFormation stack to create the dashboards | `string` | `"CI-Cloud-Intelligence-Dashboards"` | no | | [stack\_name\_collectors](#input\_stack\_name\_collectors) | The name of the CloudFormation stack to create the collectors | `string` | `"CidDataCollectionStack"` | no | @@ -62,4 +62,4 @@ | [dashboard\_bucket\_arn](#output\_dashboard\_bucket\_arn) | The name of the bucket where to store the dashboards | | [destination\_bucket\_arn](#output\_destination\_bucket\_arn) | The name of the bucket where to replicate the data from the CUR | | [destination\_bucket\_name](#output\_destination\_bucket\_name) | The name of the bucket where to replicate the data from the CUR | - \ No newline at end of file + diff --git a/modules/destination/assets/cloudformation/cudos/deploy-data-collection.yaml b/modules/destination/assets/cloudformation/cudos/deploy-data-collection.yaml index 94ccd36..990c712 100644 --- a/modules/destination/assets/cloudformation/cudos/deploy-data-collection.yaml +++ b/modules/destination/assets/cloudformation/cudos/deploy-data-collection.yaml @@ -1,13 +1,10 @@ -# -## https://raw.githubusercontent.com/awslabs/cid-framework/main/data-collection/deploy/deploy-data-collection.yaml -# -AWSTemplateFormatVersion: "2010-09-09" -Description: CID Data Collection Stack v3.5.0 +AWSTemplateFormatVersion: '2010-09-09' +Description: CID Data Collection Stack v3.7.0 Metadata: AWS::CloudFormation::Interface: ParameterGroups: - Label: - default: "Deployment parameters" + default: 'Deployment parameters' Parameters: - ManagementAccountID - RegionsInScope @@ -19,8 +16,9 @@ Metadata: - Schedule - ScheduleFrequent - CFNSourceBucket + - DataBucketsKmsKeysArns - Label: - default: "Available modules" + default: 'Available modules' Parameters: - IncludeBackupModule - IncludeBudgetsModule @@ -41,13 +39,13 @@ Metadata: - IncludeServiceQuotasModule ParameterLabels: DestinationBucket: - default: "Destination S3 bucket prefix" + default: 'Destination S3 bucket prefix' ManagementAccountRole: - default: "Management account role" + default: 'Management account role' ManagementAccountID: - default: "Comma Delimited list of Account IDs for all Management Account IDs" + default: 'Comma Delimited list of Account IDs for all Management Account IDs' MultiAccountRoleName: - default: "Multi Account Role Name" + default: 'Multi Account Role Name' Schedule: default: "Schedule can be swapped to cron, for example: cron(0 8 1,15 * ? *) for 08:00 the 1st and 15th days of the month" ScheduleFrequent: @@ -60,89 +58,66 @@ Metadata: default: "Role Prefix" CFNSourceBucket: default: "DO NOT CHANGE - A bucket that contains WA-Labs CloudFormation templates. Must be always 'aws-managed-cost-intelligence-dashboards'" + DataBucketsKmsKeysArns: + default: "" IncludeTAModule: - default: "Include AWS Trusted Advisor Data Collection Module" + default: 'Include AWS Trusted Advisor Data Collection Module' IncludeRightsizingModule: - default: "Include Rightsizing Recommendations Data Collection Module" + default: 'Include Rightsizing Recommendations Data Collection Module' IncludeCostAnomalyModule: - default: "Include Cost Anomalies Data Collection Module" + default: 'Include Cost Anomalies Data Collection Module' IncludeSupportCasesModule: - default: "Include Support Cases Data Collection Module" + default: 'Include Support Cases Data Collection Module' IncludeInventoryCollectorModule: - default: "Include Inventory Collector Module" + default: 'Include Inventory Collector Module' IncludeComputeOptimizerModule: - default: "Include AWS Compute Optimizer Data Collection Module" + default: 'Include AWS Compute Optimizer Data Collection Module' IncludeECSChargebackModule: - default: "Include ECS Chargeback Data Collection Module" + default: 'Include ECS Chargeback Data Collection Module' IncludeRDSUtilizationModule: - default: "Include RDS Utilization Data Collection Module" + default: 'Include RDS Utilization Data Collection Module' IncludeOrgDataModule: - default: "Include AWS Organization Data Collection Module" + default: 'Include AWS Organization Data Collection Module' IncludeBudgetsModule: - default: "Include AWS Budgets Collection Module" + default: 'Include AWS Budgets Collection Module' IncludeTransitGatewayModule: - default: "Include AWS TransitGateway Collection Module" + default: 'Include AWS TransitGateway Collection Module' IncludeBackupModule: - default: "Include AWS Backup Collection Module" + default: 'Include AWS Backup Collection Module' IncludeAWSFeedsModule: - default: "Include AWS Feeds Module" + default: 'Include AWS Feeds Module' IncludeHealthEventsModule: - default: "Include AWS Health Events Module" + default: 'Include AWS Health Events Module' IncludeLicenseManagerModule: - default: "Include Marketplace Licensing Collection" + default: 'Include Marketplace Licensing Collection' IncludeServiceQuotasModule: - default: "Include Service Quota Data Collection" + default: 'Include Service Quota Data Collection' IncludeQuickSightModule: - default: "Include QuickSight User Collection Module" + default: 'Include QuickSight User Collection Module' Mappings: RegionMap: # Only support regions that have QuickSight - ap-northeast-1: - { CodeBucket: aws-managed-cost-intelligence-dashboards-ap-northeast-1 } - ap-northeast-2: - { CodeBucket: aws-managed-cost-intelligence-dashboards-ap-northeast-2 } - ap-south-1: - { CodeBucket: aws-managed-cost-intelligence-dashboards-ap-south-1 } - ap-southeast-1: - { CodeBucket: aws-managed-cost-intelligence-dashboards-ap-southeast-1 } - ap-southeast-2: - { CodeBucket: aws-managed-cost-intelligence-dashboards-ap-southeast-2 } - ca-central-1: - { CodeBucket: aws-managed-cost-intelligence-dashboards-ca-central-1 } - eu-central-1: - { CodeBucket: aws-managed-cost-intelligence-dashboards-eu-central-1 } - eu-north-1: - { CodeBucket: aws-managed-cost-intelligence-dashboards-eu-north-1 } - eu-west-1: - { CodeBucket: aws-managed-cost-intelligence-dashboards-eu-west-1 } - eu-west-2: - { CodeBucket: aws-managed-cost-intelligence-dashboards-eu-west-2 } - eu-west-3: - { CodeBucket: aws-managed-cost-intelligence-dashboards-eu-west-3 } - sa-east-1: - { CodeBucket: aws-managed-cost-intelligence-dashboards-sa-east-1 } - us-east-1: - { CodeBucket: aws-managed-cost-intelligence-dashboards-us-east-1 } - us-east-2: - { CodeBucket: aws-managed-cost-intelligence-dashboards-us-east-2 } - us-west-1: - { CodeBucket: aws-managed-cost-intelligence-dashboards-us-west-1 } - us-west-2: - { CodeBucket: aws-managed-cost-intelligence-dashboards-us-west-2 } + ap-northeast-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-ap-northeast-1 } + ap-northeast-2: {CodeBucket: aws-managed-cost-intelligence-dashboards-ap-northeast-2 } + ap-south-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-ap-south-1 } + ap-southeast-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-ap-southeast-1 } + ap-southeast-2: {CodeBucket: aws-managed-cost-intelligence-dashboards-ap-southeast-2 } + ca-central-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-ca-central-1 } + eu-central-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-eu-central-1 } + eu-north-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-eu-north-1 } + eu-west-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-eu-west-1 } + eu-west-2: {CodeBucket: aws-managed-cost-intelligence-dashboards-eu-west-2 } + eu-west-3: {CodeBucket: aws-managed-cost-intelligence-dashboards-eu-west-3 } + sa-east-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-sa-east-1 } + us-east-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-us-east-1 } + us-east-2: {CodeBucket: aws-managed-cost-intelligence-dashboards-us-east-2 } + us-west-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-us-west-1 } + us-west-2: {CodeBucket: aws-managed-cost-intelligence-dashboards-us-west-2 } StepFunctionCode: - main-v3: - { - TemplatePath: cfn/data-collection/source/step-functions/main-state-machine-v3.json, - } - crawler-v1: - { - TemplatePath: cfn/data-collection/source/step-functions/crawler-state-machine-v1.json, - } - standalone-v1: - { - TemplatePath: cfn/data-collection/source/step-functions/awsfeeds-state-machine-v1.json, - } + main-v3: {TemplatePath: cfn/data-collection/source/step-functions/main-state-machine-v3.json} + crawler-v1: {TemplatePath: cfn/data-collection/source/step-functions/crawler-state-machine-v1.json} + standalone-v1: {TemplatePath: cfn/data-collection/source/step-functions/awsfeeds-state-machine-v1.json} Parameters: DestinationBucket: @@ -164,11 +139,11 @@ Parameters: Default: "Optimization-Data-Multi-Account-Role" Schedule: Type: String - Description: EventBridge schedule to trigger data collection for Trusted Advisor, Compute Optimizer, Organizations Data, Rightsizing, RDS Utilization, Inventory Collector, Transit Gateway, Backup, and ECS Chargeback modules (see docs for tailoring the schedule for each module). + Description: "EventBridge schedule to trigger data collection for Trusted Advisor, Compute Optimizer, Organizations Data, Rightsizing, RDS Utilization, Inventory Collector, Transit Gateway, Backup, and ECS Chargeback modules (see docs for tailoring the schedule for each module). Recommended value: rate(14 days). Increasing data collection frequency will trigger additional cost, avoid setting the data collection schedule to more than once per day." Default: "rate(14 days)" ScheduleFrequent: Type: String - Description: EventBridge schedule to trigger data collection for Cost Anomalies, Budgets, Support Cases and Health Events modules (see docs for tailoring the schedule for each module). + Description: "EventBridge schedule to trigger data collection for Cost Anomalies, Budgets, Support Cases and Health Events modules (see docs for tailoring the schedule for each module). Recommended value: rate(1 day). Increasing data collection frequency will trigger additional cost, avoid setting the data collection schedule to more than once per day." Default: "rate(1 day)" RegionsInScope: Type: String @@ -188,143 +163,145 @@ Parameters: Type: String Description: "DO NOT CHANGE - A bucket that contains WA-Labs CloudFormation templates. Must be always 'aws-managed-cost-intelligence-dashboards'" Default: "aws-managed-cost-intelligence-dashboards" + DataBucketsKmsKeysArns: + Type: String + Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys." + Default: "" IncludeTAModule: Type: String Description: Collects AWS Trusted Advisor recommendations data - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeRightsizingModule: Type: String Description: "Collects AWS Cost Explorer Rightsizing Recommendations" - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeCostAnomalyModule: Type: String Description: "Collects AWS Cost Explorer Cost Anomalies Recommendations" - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeSupportCasesModule: Type: String Description: "Collects AWS Support Cases Data" - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeInventoryCollectorModule: Type: String Description: Collects data about AMIs, EBS volumes and snapshots - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeComputeOptimizerModule: Type: String Description: Collects AWS Compute Optimizer service recommendations - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeECSChargebackModule: Type: String Description: Collects data which shows costs associated with ECS Tasks leveraging EC2 instances within a Cluster - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeRDSUtilizationModule: Type: String Description: Collects RDS CloudWatch metrics from your accounts - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeOrgDataModule: Type: String Description: Collects AWS Organizations data such as account Id, account name, organization parent and specified tags - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeBudgetsModule: Type: String Description: Collects AWS Budgets - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeTransitGatewayModule: Type: String Description: Collects AWS TransitGateway data - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeBackupModule: Type: String Description: Collects AWS Backup data - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeAWSFeedsModule: Type: String Description: Collects AWS Feeds data - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeHealthEventsModule: Type: String Description: Collects AWS Health Events data - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeLicenseManagerModule: Type: String Description: Collects Marketplace Licenses and Grants - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeQuickSightModule: Type: String Description: Collects Marketplace Licenses and Grants - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeServiceQuotasModule: - Type: String - Description: Collects AWS Service Quotas data - AllowedValues: ["yes", "no"] - Default: "no" + Type: String + Description: Collects AWS Service Quotas data + AllowedValues: ['yes', 'no'] + Default: 'no' Conditions: - DeployTAModule: !Equals [!Ref IncludeTAModule, "yes"] - DeployRightsizingModule: !Equals [!Ref IncludeRightsizingModule, "yes"] - DeployCostAnomalyModule: !Equals [!Ref IncludeCostAnomalyModule, "yes"] - DeploySupportCasesModule: !Equals [!Ref IncludeSupportCasesModule, "yes"] - DeployInventoryCollectorModule: - !Equals [!Ref IncludeInventoryCollectorModule, "yes"] - DeployComputeOptimizerModule: - !Equals [!Ref IncludeComputeOptimizerModule, "yes"] - DeployEcsChargebackModule: !Equals [!Ref IncludeECSChargebackModule, "yes"] - DeployRDSUtilizationModule: !Equals [!Ref IncludeRDSUtilizationModule, "yes"] - DeployOrgDataModule: !Equals [!Ref IncludeOrgDataModule, "yes"] - DeployBudgetsModule: !Equals [!Ref IncludeBudgetsModule, "yes"] - DeployTransitGatewayModule: !Equals [!Ref IncludeTransitGatewayModule, "yes"] - DeployBackupModule: !Equals [!Ref IncludeBackupModule, "yes"] - DeployAWSFeedsModule: !Equals [!Ref IncludeAWSFeedsModule, "yes"] - DeployHealthEventsModule: !Equals [!Ref IncludeHealthEventsModule, "yes"] - DeployLicenseManagerModule: !Equals [!Ref IncludeLicenseManagerModule, "yes"] - DeployQuickSightModule: !Equals [!Ref IncludeQuickSightModule, "yes"] - DeployServiceQuotasModule: !Equals [!Ref IncludeServiceQuotasModule, "yes"] + DeployTAModule: !Equals [ !Ref IncludeTAModule, "yes"] + DeployRightsizingModule: !Equals [ !Ref IncludeRightsizingModule, "yes"] + DeployCostAnomalyModule: !Equals [ !Ref IncludeCostAnomalyModule, "yes"] + DeploySupportCasesModule: !Equals [ !Ref IncludeSupportCasesModule, "yes"] + DeployInventoryCollectorModule: !Equals [ !Ref IncludeInventoryCollectorModule, "yes"] + DeployComputeOptimizerModule: !Equals [ !Ref IncludeComputeOptimizerModule, "yes"] + DeployEcsChargebackModule: !Equals [ !Ref IncludeECSChargebackModule, "yes"] + DeployRDSUtilizationModule: !Equals [ !Ref IncludeRDSUtilizationModule, "yes"] + DeployOrgDataModule: !Equals [ !Ref IncludeOrgDataModule, "yes"] + DeployBudgetsModule: !Equals [ !Ref IncludeBudgetsModule, "yes"] + DeployTransitGatewayModule: !Equals [ !Ref IncludeTransitGatewayModule, "yes"] + DeployBackupModule: !Equals [ !Ref IncludeBackupModule, "yes"] + DeployAWSFeedsModule: !Equals [ !Ref IncludeAWSFeedsModule, "yes"] + DeployHealthEventsModule: !Equals [ !Ref IncludeHealthEventsModule, "yes"] + DeployLicenseManagerModule: !Equals [ !Ref IncludeLicenseManagerModule, "yes"] + DeployQuickSightModule: !Equals [ !Ref IncludeQuickSightModule, "yes"] + DeployServiceQuotasModule: !Equals [ !Ref IncludeServiceQuotasModule, "yes"] DeployPricingModule: !Or - !Condition DeployInventoryCollectorModule - !Condition DeployRDSUtilizationModule DeployAccountCollector: !Or - Fn::Or: - - !Condition DeployTAModule - - !Condition DeployRightsizingModule - - !Condition DeployCostAnomalyModule - - !Condition DeploySupportCasesModule - - !Condition DeployInventoryCollectorModule - - !Condition DeployComputeOptimizerModule - - !Condition DeployEcsChargebackModule - - !Condition DeployRDSUtilizationModule - - !Condition DeployOrgDataModule - - !Condition DeployBudgetsModule + - !Condition DeployTAModule + - !Condition DeployRightsizingModule + - !Condition DeployCostAnomalyModule + - !Condition DeploySupportCasesModule + - !Condition DeployInventoryCollectorModule + - !Condition DeployComputeOptimizerModule + - !Condition DeployEcsChargebackModule + - !Condition DeployRDSUtilizationModule + - !Condition DeployOrgDataModule + - !Condition DeployBudgetsModule - Fn::Or: - - !Condition DeployBackupModule - - !Condition DeployTransitGatewayModule - - !Condition DeployHealthEventsModule - - !Condition DeployLicenseManagerModule - - !Condition DeployQuickSightModule - - !Condition DeployServiceQuotasModule + - !Condition DeployBackupModule + - !Condition DeployTransitGatewayModule + - !Condition DeployHealthEventsModule + - !Condition DeployLicenseManagerModule + - !Condition DeployQuickSightModule + - !Condition DeployServiceQuotasModule RegionsInScopeIsEmpty: !Equals - - !Join ["", !Split [" ", !Ref RegionsInScope]] # remove spaces + - !Join [ '', !Split [ ' ', !Ref RegionsInScope ] ] # remove spaces - "" - ProdCFNTemplateUsed: - !Equals [!Ref CFNSourceBucket, "aws-managed-cost-intelligence-dashboards"] + ProdCFNTemplateUsed: !Equals [ !Ref CFNSourceBucket, 'aws-managed-cost-intelligence-dashboards' ] + NeedDataBucketsKms: !Not [ !Equals [ !Ref DataBucketsKmsKeysArns, "" ] ] Resources: S3Bucket: - Type: "AWS::S3::Bucket" + Type: 'AWS::S3::Bucket' Properties: BucketName: !Sub "${DestinationBucket}${AWS::AccountId}" VersioningConfiguration: @@ -334,10 +311,10 @@ Resources: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 PublicAccessBlockConfiguration: - BlockPublicAcls: true - BlockPublicPolicy: true - IgnorePublicAcls: true - RestrictPublicBuckets: true + BlockPublicAcls : true + BlockPublicPolicy : true + IgnorePublicAcls : true + RestrictPublicBuckets : true LifecycleConfiguration: Rules: - Id: RemoveNonCurrentVersions @@ -345,7 +322,7 @@ Resources: NoncurrentVersionExpiration: NoncurrentDays: 7 NewerNoncurrentVersions: 1 - # ExpirationInDays: 365 # Set Expiration of all objects here (not recommended but can be useful in case of big workloads) + # ExpirationInDays: 365 # Set Expiration of all objects here (not recommended but can be useful in case of big workloads) - Id: DeleteIncompleteMultipartUploads Status: Enabled AbortIncompleteMultipartUpload: @@ -373,7 +350,9 @@ Resources: Action: s3:* Effect: Deny Principal: "*" - Resource: !Sub "${S3Bucket.Arn}/*" + Resource: + - !Sub "${S3Bucket.Arn}" + - !Sub "${S3Bucket.Arn}/*" Condition: Bool: aws:SecureTransport: false @@ -381,7 +360,9 @@ Resources: Action: s3:* Effect: Deny Principal: "*" - Resource: !Sub "${S3Bucket.Arn}/*" + Resource: + - !Sub "${S3Bucket.Arn}" + - !Sub "${S3Bucket.Arn}/*" Condition: NumericLessThan: s3:TlsVersion: 1.2 @@ -407,7 +388,7 @@ Resources: Effect: Allow Principal: Service: - - glue.amazonaws.com + - !Sub "glue.${AWS::URLSuffix}" Version: 2012-10-17 Path: / Policies: @@ -419,18 +400,23 @@ Resources: Action: - s3:ListBucket Resource: - - !Sub "arn:${AWS::Partition}:s3:::${DestinationBucket}${AWS::AccountId}" + - !Sub "arn:${AWS::Partition}:s3:::${DestinationBucket}${AWS::AccountId}" - Effect: Allow Action: - s3:GetObject Resource: - - !Sub "arn:${AWS::Partition}:s3:::${DestinationBucket}${AWS::AccountId}/*" - ## Uncomment if bucket is encrypted by Custom KMS Key - #- Effect: Allow - # Action: - # - kms:Decrypt - # Resource: - # - !Sub "arn:${AWS::Partition}:kms:${AWS::Region}:${AWS::AccountId}:key/key-id" + - !Sub "arn:${AWS::Partition}:s3:::${DestinationBucket}${AWS::AccountId}/*" + - !If + - NeedDataBucketsKms + - PolicyName: "KMS" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + - "kms:Decrypt" + Resource: !Split [ ',', !Ref DataBucketsKmsKeysArns ] + - !Ref AWS::NoValue - PolicyName: "Glue" PolicyDocument: Version: "2012-10-17" @@ -486,11 +472,12 @@ Resources: - Effect: Allow Principal: Service: - - lambda.amazonaws.com + - !Sub "lambda.${AWS::URLSuffix}" Action: - sts:AssumeRole ManagedPolicyArns: - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + LambdaAnalytics: Type: AWS::Lambda::Function Properties: @@ -563,7 +550,7 @@ Resources: - Effect: Allow Principal: Service: - - lambda.amazonaws.com + - !Sub "lambda.${AWS::URLSuffix}" Action: - sts:AssumeRole ManagedPolicyArns: @@ -587,6 +574,25 @@ Resources: - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog" - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}" - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${DatabaseName}/*" + + KmsPolicyForCidResources: + Type: AWS::IAM::Policy + Condition: NeedDataBucketsKms + Properties: + PolicyName: !Sub "${ResourcePrefix}AwsDataCollectionKmsDecryption" + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'kms:Decrypt' + Resource: !Split [ ',', !Ref DataBucketsKmsKeysArns ] + Roles: + - !Ref LambdaInitRole + - !Ref StepFunctionExecutionRole + - !Ref LambdaManageGlueTableRole + - !Ref GlueRole + LambdaInit: Type: AWS::Lambda::Function Properties: @@ -686,7 +692,7 @@ Resources: - Effect: Allow Principal: Service: - - lambda.amazonaws.com + - !Sub "lambda.${AWS::URLSuffix}" Action: - sts:AssumeRole ManagedPolicyArns: @@ -711,6 +717,7 @@ Resources: - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog" - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}" - !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${DatabaseName}/*" + LambdaManageGlueTable: Type: AWS::Lambda::Function Properties: @@ -796,16 +803,11 @@ Resources: CrawlerExecutionStepFunction: Type: AWS::StepFunctions::StateMachine Properties: - StateMachineName: !Sub "${ResourcePrefix}CrawlerExecution-StateMachine" + StateMachineName: !Sub '${ResourcePrefix}CrawlerExecution-StateMachine' StateMachineType: STANDARD RoleArn: !GetAtt StepFunctionExecutionRole.Arn DefinitionS3Location: - Bucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] + Bucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] Key: !FindInMap [StepFunctionCode, crawler-v1, TemplatePath] StepFunctionExecutionRole: @@ -819,7 +821,7 @@ Resources: - Effect: Allow Principal: Service: - - states.amazonaws.com + - !Sub "states.${AWS::URLSuffix}" Action: - sts:AssumeRole Policies: @@ -831,7 +833,7 @@ Resources: Action: - glue:StartCrawler - glue:GetCrawler - Resource: !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/${ResourcePrefix}*Crawler*" + Resource: !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/${ResourcePrefix}*Crawler*' - PolicyName: InvokeCollectionLambda PolicyDocument: Version: 2012-10-17 @@ -840,7 +842,7 @@ Resources: Action: - lambda:InvokeFunction Resource: - - !Sub "arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:${ResourcePrefix}*Lambda*" + - !Sub 'arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:${ResourcePrefix}*Lambda*' - PolicyName: PolicyForSyncronousExecution PolicyDocument: Version: 2012-10-17 @@ -851,19 +853,19 @@ Resources: - events:DescribeRule - events:PutRule Resource: - - !Sub "arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule" + - !Sub 'arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule' - Effect: Allow Action: - states:StartExecution Resource: - - !Sub "arn:${AWS::Partition}:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}*-StateMachine" + - !Sub 'arn:${AWS::Partition}:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}*-StateMachine' - Effect: Allow Action: - states:DescribeExecution - states:StopExecution Resource: - - !Sub "arn:${AWS::Partition}:states:${AWS::Region}:${AWS::AccountId}:execution:*:*" - - !Sub "arn:${AWS::Partition}:states:${AWS::Region}:${AWS::AccountId}:express:*:*:*" + - !Sub 'arn:${AWS::Partition}:states:${AWS::Region}:${AWS::AccountId}:execution:*:*' + - !Sub 'arn:${AWS::Partition}:states:${AWS::Region}:${AWS::AccountId}:express:*:*:*' - PolicyName: "S3-ReadOnlyAccess" #Used for getting summary record list for map iterations to retrieve detail API calls PolicyDocument: Version: "2012-10-17" @@ -871,10 +873,10 @@ Resources: - Effect: "Allow" Action: - "s3:GetObject" - Resource: !Sub "${S3Bucket.Arn}/*" + Resource: !Sub '${S3Bucket.Arn}/*' StepFunctionExecutionRoleInvokeAccountCollectorPolicy: - Type: "AWS::IAM::Policy" + Type: 'AWS::IAM::Policy' Condition: DeployAccountCollector Properties: PolicyName: InvokeAccountCollector @@ -900,7 +902,7 @@ Resources: - Effect: Allow Principal: Service: - - scheduler.amazonaws.com + - !Sub "scheduler.${AWS::URLSuffix}" Action: - sts:AssumeRole Policies: @@ -927,9 +929,10 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployTAModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-trusted-advisor.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-trusted-advisor.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn GlueRoleARN: !GetAtt GlueRole.Arn @@ -938,14 +941,8 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -953,9 +950,10 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployRightsizingModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-cost-explorer-rightsizing.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-cost-explorer-rightsizing.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}" @@ -964,14 +962,8 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -979,9 +971,10 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployCostAnomalyModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-cost-anomaly.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-cost-anomaly.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}" @@ -990,25 +983,20 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn - + SupportCasesModule: Type: AWS::CloudFormation::Stack Condition: DeploySupportCasesModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-support-cases.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-support-cases.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}" @@ -1017,14 +1005,8 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1032,9 +1014,10 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployBackupModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-backup.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-backup.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}" @@ -1043,14 +1026,8 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1058,25 +1035,20 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployInventoryCollectorModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-inventory.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-inventory.yaml" Parameters: DatabaseName: !Ref DatabaseName DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns GlueRoleARN: !GetAtt GlueRole.Arn MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}" Schedule: !Ref Schedule ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1084,24 +1056,20 @@ Resources: Fn::If: - RegionsInScopeIsEmpty - !Sub "${AWS::Region}" - - !Join ["", !Split [" ", !Ref RegionsInScope]] # remove spaces + - !Join [ '', !Split [ ' ', !Ref RegionsInScope ] ] # remove spaces PricingModule: Type: AWS::CloudFormation::Stack Condition: DeployPricingModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-pricing.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-pricing.yaml" Parameters: DatabaseName: !Ref DatabaseName DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns GlueRoleARN: !GetAtt GlueRole.Arn - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] Schedule: !Ref Schedule ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn @@ -1111,35 +1079,30 @@ Resources: Fn::If: - RegionsInScopeIsEmpty - !Sub "${AWS::Region}" - - !Join ["", !Split [" ", !Ref RegionsInScope]] # remove spaces + - !Join [ '', !Split [ ' ', !Ref RegionsInScope ] ] # remove spaces ComputeOptimizerModule: Type: AWS::CloudFormation::Stack Condition: DeployComputeOptimizerModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-compute-optimizer.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-compute-optimizer.yaml" Parameters: DestinationBucket: !Ref S3Bucket ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}" ManagementAccountID: !Ref ManagementAccountID + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns Schedule: !Ref Schedule ResourcePrefix: !Ref ResourcePrefix - BucketPrefix: !Ref DestinationBucket + BucketPrefix: !Ref DestinationBucket RegionsInScope: Fn::If: - RegionsInScopeIsEmpty - !Sub "${AWS::Region}" - - !Join ["", !Split [" ", !Ref RegionsInScope]] # remove spaces + - !Join [ '', !Split [ ' ', !Ref RegionsInScope ] ] # remove spaces LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1147,9 +1110,10 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployEcsChargebackModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-ecs-chargeback.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-ecs-chargeback.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn GlueRoleARN: !GetAtt GlueRole.Arn @@ -1158,29 +1122,24 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn RegionsInScope: Fn::If: - RegionsInScopeIsEmpty - !Sub "${AWS::Region}" - - !Join ["", !Split [" ", !Ref RegionsInScope]] # remove spaces + - !Join [ '', !Split [ ' ', !Ref RegionsInScope ] ] # remove spaces RDSUsageModule: Type: AWS::CloudFormation::Stack Condition: DeployRDSUtilizationModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-rds-usage.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-rds-usage.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn GlueRoleARN: !GetAtt GlueRole.Arn @@ -1189,29 +1148,24 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn RegionsInScope: Fn::If: - RegionsInScopeIsEmpty - !Sub "${AWS::Region}" - - !Join ["", !Split [" ", !Ref RegionsInScope]] # remove spaces + - !Join [ '', !Split [ ' ', !Ref RegionsInScope ] ] # remove spaces OrgDataModule: Type: AWS::CloudFormation::Stack Condition: DeployOrgDataModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-organization.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-organization.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn GlueRoleARN: !GetAtt GlueRole.Arn @@ -1220,14 +1174,8 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1235,25 +1183,20 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployBudgetsModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-budgets.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-budgets.yaml" Parameters: DatabaseName: !Ref DatabaseName DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn GlueRoleARN: !GetAtt GlueRole.Arn + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}" Schedule: !Ref ScheduleFrequent ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1261,9 +1204,10 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployTransitGatewayModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-transit-gateway.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-transit-gateway.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn GlueRoleARN: !GetAtt GlueRole.Arn @@ -1272,43 +1216,32 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn RegionsInScope: Fn::If: - RegionsInScopeIsEmpty - !Sub "${AWS::Region}" - - !Join ["", !Split [" ", !Ref RegionsInScope]] # remove spaces + - !Join [ '', !Split [ ' ', !Ref RegionsInScope ] ] # remove spaces AWSFeedsModule: Type: AWS::CloudFormation::Stack Condition: DeployAWSFeedsModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-aws-feeds.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-aws-feeds.yaml" Parameters: DatabaseName: !Ref DatabaseName DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns Schedule: !Ref ScheduleFrequent GlueRoleARN: !GetAtt GlueRole.Arn ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, standalone-v1, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, standalone-v1, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1316,9 +1249,10 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployHealthEventsModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-health-events.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-health-events.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn Schedule: !Ref ScheduleFrequent @@ -1327,14 +1261,8 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1342,9 +1270,10 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployLicenseManagerModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-license-manager.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-license-manager.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}" @@ -1353,14 +1282,8 @@ Resources: ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1375,47 +1298,37 @@ Resources: DestinationBucketARN: !GetAtt S3Bucket.Arn MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}" Schedule: !Ref ScheduleFrequent + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns GlueRoleARN: !GetAtt GlueRole.Arn ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, main-v3, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn RegionsInScope: Fn::If: - RegionsInScopeIsEmpty - !Sub "${AWS::Region}" - - !Join ["", !Split [" ", !Ref RegionsInScope]] # remove spaces + - !Join [ '', !Split [ ' ', !Ref RegionsInScope ] ] # remove spaces QuickSightModule: Type: AWS::CloudFormation::Stack Condition: DeployQuickSightModule Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-quicksight.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-quicksight.yaml" Parameters: DatabaseName: !Ref DatabaseName + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn Schedule: !Ref ScheduleFrequent GlueRoleARN: !GetAtt GlueRole.Arn ResourcePrefix: !Ref ResourcePrefix LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn - CodeBucket: - !If [ - ProdCFNTemplateUsed, - !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], - !Ref CFNSourceBucket, - ] - StepFunctionTemplate: - !FindInMap [StepFunctionCode, standalone-v1, TemplatePath] + CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] + StepFunctionTemplate: !FindInMap [StepFunctionCode, standalone-v1, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1423,21 +1336,22 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployAccountCollector Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/account-collector.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/account-collector.yaml" Parameters: ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}" ManagementAccountID: !Ref ManagementAccountID ResourcePrefix: !Ref ResourcePrefix DestinationBucket: !Ref S3Bucket DestinationBucketARN: !GetAtt S3Bucket.Arn + DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns DataCollectionReadAccess: Type: AWS::IAM::ManagedPolicy Properties: ManagedPolicyName: !Sub ${ResourcePrefix}DataCollectionReadAccess - Description: "Policy for QuickSight to allow DataCollection access" + Description: 'Policy for QuickSight to allow DataCollection access' PolicyDocument: - Version: "2012-10-17" + Version: '2012-10-17' Statement: - Sid: AllowGlue Effect: Allow @@ -1464,6 +1378,14 @@ Resources: - s3:GetObjectVersion Resource: - !Sub ${S3Bucket.Arn}/* + - !If + - NeedDataBucketsKms + - Sid: AllowKmsDecrypt + Effect: "Allow" + Action: + - "kms:Decrypt" + Resource: !Split [ ',', !Ref DataBucketsKmsKeysArns ] + - !Ref AWS::NoValue Outputs: Bucket: diff --git a/modules/destination/locals.tf b/modules/destination/locals.tf index 4d00758..3f164b3 100644 --- a/modules/destination/locals.tf +++ b/modules/destination/locals.tf @@ -9,7 +9,6 @@ locals { ## The URL for the s3 bucket containing cloudformation scripts bucket_url = format("https://%s.s3.%s.amazonaws.com", var.cloudformation_bucket_name, local.region) ## Indicates if we should provision the quicksight admin user - enable_admin = var.enable_quicksight_admin && var.quicksight_admin_email != null ## Is the user mappings for the quicksight groups user_group_mappings = merge([ diff --git a/modules/destination/main.tf b/modules/destination/main.tf index 24a80f0..6e43674 100644 --- a/modules/destination/main.tf +++ b/modules/destination/main.tf @@ -104,7 +104,7 @@ resource "aws_quicksight_account_subscription" "subscription" { ## Provision a administator user in quicksight resource "aws_quicksight_user" "admin" { - count = local.enable_admin ? 1 : 0 + count = var.enable_quicksight_admin ? 1 : 0 email = var.quicksight_admin_email identity_type = "QUICKSIGHT" @@ -264,7 +264,7 @@ module "dashboards" { "DeployTAODashboard" = var.enable_tao_dashboard ? "yes" : "no" "PrerequisitesQuickSight" = var.enable_prerequisites_quicksight ? "yes" : "no" "PrerequisitesQuickSightPermissions" = var.enable_prerequisites_quicksight_permissions ? "yes" : "no" - "QuickSightUser" = var.quicksights_username + "QuickSightUser" = var.quicksight_dashboard_owner } depends_on = [ diff --git a/modules/destination/quicksights.tf b/modules/destination/quicksights.tf index 7b9ef22..bf4b75a 100644 --- a/modules/destination/quicksights.tf +++ b/modules/destination/quicksights.tf @@ -13,11 +13,11 @@ resource "aws_quicksight_user" "users" { for_each = var.quicksight_users email = each.key - iam_arn = each.value.identity_type == "IAM" ? aws_iam_role.cudos_sso[0].arn : null + iam_arn = each.value.identity_type == "IAM" ? try(aws_iam_role.cudos_sso[0].arn, null) : null identity_type = each.value.identity_type namespace = try(each.value.namespace, "default") session_name = each.value.identity_type == "IAM" ? each.key : null - user_name = each.value.identity_type == "QUICKSIGHT" ? try(each.value.user_name, null) : null + user_name = each.value.identity_type == "QUICKSIGHT" ? try(split("@", each.key)[0], each.key) : null user_role = try(each.value.role, null) lifecycle { diff --git a/modules/destination/variables.tf b/modules/destination/variables.tf index b645301..c1cb990 100644 --- a/modules/destination/variables.tf +++ b/modules/destination/variables.tf @@ -81,7 +81,7 @@ variable "dashboards_bucket_name" { } variable "enable_quicksight_admin" { - description = "Enable the creation of an admin user (var.quicksights_username) in QuickSight" + description = "Enable the creation of an admin user (var.quicksight_admin_username) in QuickSight" type = bool default = true } @@ -93,7 +93,7 @@ variable "quicksight_admin_username" { } variable "quicksight_admin_email" { - description = "The email address for the QuickSight admin user" + description = "The email address for the QuickSight admin user. Required if var.create_quicksight_admin_user is true" type = string default = null } @@ -239,7 +239,7 @@ variable "enable_prerequisites_quicksight_permissions" { variable "saml_metadata" { description = "The configuration for the SAML identity provider" type = string - default = null + default = "" } variable "quicksight_groups" { @@ -252,8 +252,8 @@ variable "quicksight_groups" { default = {} } -variable "quicksights_username" { - description = "The username for the QuickSight user" +variable "quicksight_dashboard_owner" { + description = "The username for the QuickSight user who will own the dashboards. This user needs to exist. By default, it will be the admin user which is created by the module." type = string default = "admin" } @@ -261,7 +261,7 @@ variable "quicksights_username" { variable "quicksight_users" { description = "Map of user accounts to be registered in QuickSight" type = map(object({ - identity_type = optional(string, "IAM") + identity_type = string namespace = optional(string, "default") role = optional(string, "READER") })) diff --git a/modules/source/assets/cloudformation/cudos/deploy-data-read-permissions.yaml b/modules/source/assets/cloudformation/cudos/deploy-data-read-permissions.yaml index 49afa30..fc1bc05 100644 --- a/modules/source/assets/cloudformation/cudos/deploy-data-read-permissions.yaml +++ b/modules/source/assets/cloudformation/cudos/deploy-data-read-permissions.yaml @@ -1,8 +1,5 @@ -# -## https://github.com/awslabs/cid-framework/blob/main/data-collection/deploy/deploy-data-read-permissions.yaml -# AWSTemplateFormatVersion: "2010-09-09" -Description: CID Data Collection - All-in-One for Management Account v3.5.0 +Description: CID Data Collection - All-in-One for Management Account v3.7.0 Metadata: AWS::CloudFormation::Interface: ParameterGroups: @@ -109,73 +106,73 @@ Parameters: IncludeBudgetsModule: Type: String Description: Collects budgets from your accounts - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeComputeOptimizerModule: Type: String Description: Collects AWS Compute Optimizer service recommendations - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeCostAnomalyModule: Type: String Description: "Collects AWS Cost Explorer Cost Anomalies Recommendations" - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeSupportCasesModule: Type: String Description: "Collects AWS Support Cases data" - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeECSChargebackModule: Type: String Description: Collects data which shows costs associated with ECS Tasks leveraging EC2 instances within a Cluster - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeInventoryCollectorModule: Type: String Description: Collects data about AMIs, EBS volumes and snapshots - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeRDSUtilizationModule: Type: String Description: Collects RDS CloudWatch metrics from your accounts - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeRightsizingModule: Type: String Description: "Collects AWS Cost Explorer Rightsizing Recommendations" - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeTAModule: Type: String Description: Collects AWS Trusted Advisor recommendations data - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeTransitGatewayModule: Type: String Description: Collects TransitGateway from your accounts - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeBackupModule: Type: String Description: Collects AWS Backup events from your accounts - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeHealthEventsModule: Type: String Description: Collects AWS Health Events from your accounts - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeLicenseManagerModule: Type: String Description: Collects Marketplace Licensing information - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' IncludeServiceQuotasModule: Type: String Description: Collects Service Quotas information - AllowedValues: ["yes", "no"] - Default: "no" + AllowedValues: ['yes', 'no'] + Default: 'no' Conditions: DeployModuleReadInMgmt: !Equals [!Ref AllowModuleReadInMgmt, "yes"] @@ -184,7 +181,7 @@ Resources: DataCollectorMgmtAccountReadStack: Type: AWS::CloudFormation::Stack Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/deploy-in-management-account.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/deploy-in-management-account.yaml" Parameters: DataCollectionAccountID: !Ref DataCollectionAccountID ManagementAccountRole: !Ref ManagementAccountRole @@ -201,7 +198,7 @@ Resources: Type: AWS::CloudFormation::Stack Condition: DeployModuleReadInMgmt Properties: - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/deploy-in-linked-account.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/deploy-in-linked-account.yaml" Parameters: DataCollectionAccountID: !Ref DataCollectionAccountID MultiAccountRoleName: !Ref MultiAccountRoleName @@ -217,7 +214,7 @@ Resources: DataCollectorOrgAccountModulesReadStackSet: Type: AWS::CloudFormation::StackSet Properties: - Description: "StackSet in charge of deploying read roles across organization accounts v3.5.0" + Description: "StackSet in charge of deploying read roles across organization accounts v3.7.0" PermissionModel: SERVICE_MANAGED AutoDeployment: Enabled: true @@ -260,4 +257,4 @@ Resources: - CAPABILITY_IAM - CAPABILITY_NAMED_IAM StackSetName: !Sub "StackSet-${AWS::AccountId}-OptimizationDataRole" - TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/deploy-in-linked-account.yaml" + TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/deploy-in-linked-account.yaml"