From 642d73182ee39702451555ee3b7e498fe88dc4c6 Mon Sep 17 00:00:00 2001 From: douniwan5788 Date: Tue, 2 Jul 2024 12:02:23 +0800 Subject: [PATCH 1/2] fix: prefix list call 'incorrect version number" --- lambda/update_aws_ip_ranges.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lambda/update_aws_ip_ranges.py b/lambda/update_aws_ip_ranges.py index f494f89..a511788 100644 --- a/lambda/update_aws_ip_ranges.py +++ b/lambda/update_aws_ip_ranges.py @@ -574,6 +574,8 @@ def create_prefix_list(client: Any, prefix_list_name: str, prefix_list_ip_versio time.sleep(seconds_to_wait) wait_prefix_list: dict[str, Any] = get_prefix_list_by_id(client, response['PrefixList']['PrefixListId']) if wait_prefix_list['State'] in {'create-complete', 'modify-complete'}: + # update response['PrefixList']['Version'] after modify-complete + response['PrefixList'] = wait_prefix_list break else: # Else doesn't execute if exit via break @@ -692,6 +694,8 @@ def update_prefix_list(client: Any, prefix_list_name: str, prefix_list: dict[str time.sleep(seconds_to_wait) wait_prefix_list: dict[str, Any] = get_prefix_list_by_id(client, response['PrefixList']['PrefixListId']) if wait_prefix_list['State'] in {'modify-complete'}: + # update response['PrefixList']['Version'] after modify-complete + response['PrefixList'] = wait_prefix_list break else: # Else doesn't execute if exit via break @@ -704,8 +708,8 @@ def update_prefix_list(client: Any, prefix_list_name: str, prefix_list: dict[str PrefixListId=response['PrefixList']['PrefixListId'], CurrentVersion=response['PrefixList']['Version'], PrefixListName=response['PrefixList']['PrefixListName'], - AddEntries=entries_to_add[0:100], - RemoveEntries=entries_to_remove[0:100] + AddEntries=entries_to_add[index:index+100], + RemoveEntries=entries_to_remove[index:index+100] ) logging.info(f'Updated VPC Prefix List "{prefix_list_name}"') logging.info(f'Response: {response}') From 8cd2565d3de4c480163ee410a83b8d36976a6b79 Mon Sep 17 00:00:00 2001 From: douniwan5788 Date: Tue, 2 Jul 2024 12:06:46 +0800 Subject: [PATCH 2/2] fix: prefix list entries limit of 1000(1k), split into chunks --- cloudformation/template.yml | 4 ++++ lambda/services.json | 6 +++-- lambda/update_aws_ip_ranges.py | 42 ++++++++++++++++++++++------------ 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/cloudformation/template.yml b/cloudformation/template.yml index 4572eac..227f4fd 100644 --- a/cloudformation/template.yml +++ b/cloudformation/template.yml @@ -160,7 +160,9 @@ Resources: StringLike: 'aws:ResourceTag/Name': - 'aws-ip-ranges-*-ipv4' + - 'aws-ip-ranges-*-ipv4-continued-?' - 'aws-ip-ranges-*-ipv6' + - 'aws-ip-ranges-*-ipv6-continued-?' - Effect: 'Allow' Action: @@ -174,7 +176,9 @@ Resources: StringLike: 'aws:RequestTag/Name': - 'aws-ip-ranges-*-ipv4' + - 'aws-ip-ranges-*-ipv4-continued-?' - 'aws-ip-ranges-*-ipv6' + - 'aws-ip-ranges-*-ipv6-continued-?' 'ForAllValues:StringEquals': 'aws:TagKeys': - 'Name' diff --git a/lambda/services.json b/lambda/services.json index dbfed3b..2a6e17b 100644 --- a/lambda/services.json +++ b/lambda/services.json @@ -5,7 +5,8 @@ "Regions": ["sa-east-1"], "PrefixList": { "Enable": true, - "Summarize": false + "Summarize": false, + "ChunkSize": 998 }, "WafIPSet": { "Enable": true, @@ -31,7 +32,8 @@ "Regions": ["sa-east-1"], "PrefixList": { "Enable": true, - "Summarize": false + "Summarize": false, + "ChunkSize": 500 }, "WafIPSet": { "Enable": true, diff --git a/lambda/update_aws_ip_ranges.py b/lambda/update_aws_ip_ranges.py index a511788..46c31cf 100644 --- a/lambda/update_aws_ip_ranges.py +++ b/lambda/update_aws_ip_ranges.py @@ -5,6 +5,7 @@ import hashlib import ipaddress +import itertools import json import logging import os @@ -423,7 +424,7 @@ def get_ip_set_entries(client: Any, ipset_name: str, ipset_scope: str, ipset_id: ### VPC Prefix List -def manage_prefix_list(client: Any, vpc_prefix_lists: dict[str, dict], service_name: str, service_ranges: dict[str, ServiceIPRange], should_summarize: bool) -> dict[str, list[str]]: +def manage_prefix_list(client: Any, vpc_prefix_lists: dict[str, dict], service_name: str, service_ranges: dict[str, ServiceIPRange], should_summarize: bool, chunk_size: int) -> dict[str, list[str]]: """Create or Update VPC Prefix List""" logging.info('manage_prefix_list start') logging.debug(f'Parameter client: {client}') @@ -445,18 +446,24 @@ def manage_prefix_list(client: Any, vpc_prefix_lists: dict[str, dict], service_n if should_summarize and (len(address_list) > 1): address_list = service_ranges[service_name].asdict()[ip_version].summarized() - prefix_list_name: str = f"{RESOURCE_NAME_PREFIX}-{service_name.lower().replace('_', '-')}-{ip_version}" - if prefix_list_name in vpc_prefix_lists: - # Prefix List exists, so will update it - logging.debug(f'VPC Prefix List "{prefix_list_name}" found. Will update it.') - updated: bool = update_prefix_list(client, prefix_list_name, vpc_prefix_lists[prefix_list_name], address_list) - if updated: - prefix_list_names['updated'].append(prefix_list_name) - else: - # Prefix List not found, so will create it - logging.debug(f'VPC Prefix List "{prefix_list_name}" not found. Will create it.') - create_prefix_list(client, prefix_list_name, ip_version.upper(), address_list) - prefix_list_names['created'].append(prefix_list_name) + base_prefix_list_name: str = f"{RESOURCE_NAME_PREFIX}-{service_name.lower().replace('_', '-')}-{ip_version}" + for index, address_list_chunk in enumerate(itertools.batched(address_list, chunk_size)): + if index == 0: + prefix_list_name = base_prefix_list_name + else: + prefix_list_name = f"{base_prefix_list_name}-continued-{index}" + + if prefix_list_name in vpc_prefix_lists: + # Prefix List exists, so will update it + logging.debug(f'VPC Prefix List "{prefix_list_name}" found. Will update it.') + updated: bool = update_prefix_list(client, prefix_list_name, vpc_prefix_lists[prefix_list_name], address_list_chunk) + if updated: + prefix_list_names['updated'].append(prefix_list_name) + else: + # Prefix List not found, so will create it + logging.debug(f'VPC Prefix List "{prefix_list_name}" not found. Will create it.') + create_prefix_list(client, prefix_list_name, ip_version.upper(), address_list_chunk) + prefix_list_names['created'].append(prefix_list_name) logging.debug(f'Function return: {prefix_list_names}') logging.info('manage_prefix_list end') @@ -526,7 +533,7 @@ def create_prefix_list(client: Any, prefix_list_name: str, prefix_list_ip_versio logging.debug(f'Prefix List entries: {prefix_list_entries}') # Add 10 enties extra when create Prefix List for future expansion - max_entries: int = len(prefix_list_entries) + 10 + max_entries: int = min(len(prefix_list_entries) + 10, 1000) logging.info(f'Creating VPC Prefix List "{prefix_list_name}" with max entries "{max_entries}" with address family "{prefix_list_ip_version}" with {len(address_list)} CIDRs. List: {address_list}') logging.info(f'VPC Prefix List entries in this call: {prefix_list_entries[0:100]}') @@ -864,7 +871,12 @@ def lambda_handler(event, context): vpc_prefix_lists = list_prefix_lists(ec2_client) should_summarize = config_service['PrefixList']['Summarize'] - prefix_list_names: dict[str, list[str]] = manage_prefix_list(ec2_client, vpc_prefix_lists, service_name, service_ranges, should_summarize) + + # left 2 for default route态rule + chunk_size :int = 998 + if "ChunkSize" in config_service["PrefixList"]: + chunk_size = config_service["PrefixList"]["ChunkSize"] + prefix_list_names: dict[str, list[str]] = manage_prefix_list(ec2_client, vpc_prefix_lists, service_name, service_ranges, should_summarize, chunk_size) resource_names['PrefixList']['created'] += prefix_list_names['created'] resource_names['PrefixList']['updated'] += prefix_list_names['updated'] except Exception as error: