Skip to content

Increase prefix list entry limit to more than 1000 and fix bugs #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Increase prefix list entry limit to more than 1000 and fix bugs #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
642d731
fix: prefix list call 'incorrect version number"
douniwan5788 Jul 2, 2024
8cd2565
fix: prefix list entries limit of 1000(1k), split into chunks
douniwan5788 Jul 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cloudformation/template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,9 @@ Resources:
StringLike:
'aws:ResourceTag/Name':
- 'aws-ip-ranges-*-ipv4'
- 'aws-ip-ranges-*-ipv4-continued-?'
- 'aws-ip-ranges-*-ipv6'
- 'aws-ip-ranges-*-ipv6-continued-?'

- Effect: 'Allow'
Action:
Expand All @@ -174,7 +176,9 @@ Resources:
StringLike:
'aws:RequestTag/Name':
- 'aws-ip-ranges-*-ipv4'
- 'aws-ip-ranges-*-ipv4-continued-?'
- 'aws-ip-ranges-*-ipv6'
- 'aws-ip-ranges-*-ipv6-continued-?'
'ForAllValues:StringEquals':
'aws:TagKeys':
- 'Name'
Expand Down
6 changes: 4 additions & 2 deletions lambda/services.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
"Regions": ["sa-east-1"],
"PrefixList": {
"Enable": true,
"Summarize": false
"Summarize": false,
"ChunkSize": 998
},
"WafIPSet": {
"Enable": true,
Expand All @@ -31,7 +32,8 @@
"Regions": ["sa-east-1"],
"PrefixList": {
"Enable": true,
"Summarize": false
"Summarize": false,
"ChunkSize": 500
},
"WafIPSet": {
"Enable": true,
Expand Down
50 changes: 33 additions & 17 deletions lambda/update_aws_ip_ranges.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import hashlib
import ipaddress
import itertools
import json
import logging
import os
Expand Down Expand Up @@ -423,7 +424,7 @@ def get_ip_set_entries(client: Any, ipset_name: str, ipset_scope: str, ipset_id:


### VPC Prefix List
def manage_prefix_list(client: Any, vpc_prefix_lists: dict[str, dict], service_name: str, service_ranges: dict[str, ServiceIPRange], should_summarize: bool) -> dict[str, list[str]]:
def manage_prefix_list(client: Any, vpc_prefix_lists: dict[str, dict], service_name: str, service_ranges: dict[str, ServiceIPRange], should_summarize: bool, chunk_size: int) -> dict[str, list[str]]:
"""Create or Update VPC Prefix List"""
logging.info('manage_prefix_list start')
logging.debug(f'Parameter client: {client}')
Expand All @@ -445,18 +446,24 @@ def manage_prefix_list(client: Any, vpc_prefix_lists: dict[str, dict], service_n
if should_summarize and (len(address_list) > 1):
address_list = service_ranges[service_name].asdict()[ip_version].summarized()

prefix_list_name: str = f"{RESOURCE_NAME_PREFIX}-{service_name.lower().replace('_', '-')}-{ip_version}"
if prefix_list_name in vpc_prefix_lists:
# Prefix List exists, so will update it
logging.debug(f'VPC Prefix List "{prefix_list_name}" found. Will update it.')
updated: bool = update_prefix_list(client, prefix_list_name, vpc_prefix_lists[prefix_list_name], address_list)
if updated:
prefix_list_names['updated'].append(prefix_list_name)
else:
# Prefix List not found, so will create it
logging.debug(f'VPC Prefix List "{prefix_list_name}" not found. Will create it.')
create_prefix_list(client, prefix_list_name, ip_version.upper(), address_list)
prefix_list_names['created'].append(prefix_list_name)
base_prefix_list_name: str = f"{RESOURCE_NAME_PREFIX}-{service_name.lower().replace('_', '-')}-{ip_version}"
for index, address_list_chunk in enumerate(itertools.batched(address_list, chunk_size)):
if index == 0:
prefix_list_name = base_prefix_list_name
else:
prefix_list_name = f"{base_prefix_list_name}-continued-{index}"

if prefix_list_name in vpc_prefix_lists:
# Prefix List exists, so will update it
logging.debug(f'VPC Prefix List "{prefix_list_name}" found. Will update it.')
updated: bool = update_prefix_list(client, prefix_list_name, vpc_prefix_lists[prefix_list_name], address_list_chunk)
if updated:
prefix_list_names['updated'].append(prefix_list_name)
else:
# Prefix List not found, so will create it
logging.debug(f'VPC Prefix List "{prefix_list_name}" not found. Will create it.')
create_prefix_list(client, prefix_list_name, ip_version.upper(), address_list_chunk)
prefix_list_names['created'].append(prefix_list_name)

logging.debug(f'Function return: {prefix_list_names}')
logging.info('manage_prefix_list end')
Expand Down Expand Up @@ -526,7 +533,7 @@ def create_prefix_list(client: Any, prefix_list_name: str, prefix_list_ip_versio
logging.debug(f'Prefix List entries: {prefix_list_entries}')

# Add 10 enties extra when create Prefix List for future expansion
max_entries: int = len(prefix_list_entries) + 10
max_entries: int = min(len(prefix_list_entries) + 10, 1000)

logging.info(f'Creating VPC Prefix List "{prefix_list_name}" with max entries "{max_entries}" with address family "{prefix_list_ip_version}" with {len(address_list)} CIDRs. List: {address_list}')
logging.info(f'VPC Prefix List entries in this call: {prefix_list_entries[0:100]}')
Expand Down Expand Up @@ -574,6 +581,8 @@ def create_prefix_list(client: Any, prefix_list_name: str, prefix_list_ip_versio
time.sleep(seconds_to_wait)
wait_prefix_list: dict[str, Any] = get_prefix_list_by_id(client, response['PrefixList']['PrefixListId'])
if wait_prefix_list['State'] in {'create-complete', 'modify-complete'}:
# update response['PrefixList']['Version'] after modify-complete
response['PrefixList'] = wait_prefix_list
break
else:
# Else doesn't execute if exit via break
Expand Down Expand Up @@ -692,6 +701,8 @@ def update_prefix_list(client: Any, prefix_list_name: str, prefix_list: dict[str
time.sleep(seconds_to_wait)
wait_prefix_list: dict[str, Any] = get_prefix_list_by_id(client, response['PrefixList']['PrefixListId'])
if wait_prefix_list['State'] in {'modify-complete'}:
# update response['PrefixList']['Version'] after modify-complete
response['PrefixList'] = wait_prefix_list
break
else:
# Else doesn't execute if exit via break
Expand All @@ -704,8 +715,8 @@ def update_prefix_list(client: Any, prefix_list_name: str, prefix_list: dict[str
PrefixListId=response['PrefixList']['PrefixListId'],
CurrentVersion=response['PrefixList']['Version'],
PrefixListName=response['PrefixList']['PrefixListName'],
AddEntries=entries_to_add[0:100],
RemoveEntries=entries_to_remove[0:100]
AddEntries=entries_to_add[index:index+100],
RemoveEntries=entries_to_remove[index:index+100]
)
logging.info(f'Updated VPC Prefix List "{prefix_list_name}"')
logging.info(f'Response: {response}')
Expand Down Expand Up @@ -860,7 +871,12 @@ def lambda_handler(event, context):
vpc_prefix_lists = list_prefix_lists(ec2_client)

should_summarize = config_service['PrefixList']['Summarize']
prefix_list_names: dict[str, list[str]] = manage_prefix_list(ec2_client, vpc_prefix_lists, service_name, service_ranges, should_summarize)

# left 2 for default route、rule
chunk_size :int = 998
if "ChunkSize" in config_service["PrefixList"]:
chunk_size = config_service["PrefixList"]["ChunkSize"]
prefix_list_names: dict[str, list[str]] = manage_prefix_list(ec2_client, vpc_prefix_lists, service_name, service_ranges, should_summarize, chunk_size)
resource_names['PrefixList']['created'] += prefix_list_names['created']
resource_names['PrefixList']['updated'] += prefix_list_names['updated']
except Exception as error:
Expand Down