Skip to content

Commit

Permalink
Replace boto with boto3 (#167)
Browse files Browse the repository at this point in the history
  • Loading branch information
asalajan authored Jan 13, 2025
1 parent 60d3aa8 commit 1f4826c
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 135 deletions.
15 changes: 8 additions & 7 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,18 +1,19 @@
simpledi==0.4.1
awscli==1.29.12
boto3==1.28.12
boto==2.49.0
ansible==8.5.0
awscli==1.32.6
boto3==1.34.6
botocore==1.34.6
urllib3==2.0.7
ansible==8.7.0
azure-common==1.1.28
azure==4.0.0
msrestazure==0.6.4
Jinja2==3.1.2
Jinja2==3.1.4
hashmerge
python-consul
hvac==1.1.1
hvac==1.2.1
passgen
inflection==0.5.1
kubernetes==26.1.0
himl==0.15.0
himl==0.15.2
six
GitPython==3.1.*
2 changes: 1 addition & 1 deletion src/ops/cli/inventory.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def run(self, args, extra_args):
group_names = [group.name for group in host.get_groups()]
group_names = sorted(group_names)
group_string = ", ".join(group_names)
host_id = host.vars.get('ec2_id', '')
host_id = host.vars.get('ec2_InstanceId', '')
if host_id != '':
name_and_id = "%s -- %s" % (stringc(host.name,
'blue'), stringc(host_id, 'blue'))
Expand Down
205 changes: 90 additions & 115 deletions src/ops/inventory/ec2inventory.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,26 +11,24 @@
import json
import re
import sys
import os

import boto
from boto import ec2
from boto.pyami.config import Config

from six import iteritems, string_types, integer_types
import boto3
from botocore.exceptions import NoRegionError, NoCredentialsError, PartialCredentialsError


class Ec2Inventory(object):
def _empty_inventory(self):
@staticmethod
def _empty_inventory():
return {"_meta": {"hostvars": {}}}

def __init__(self, boto_profile, regions, filters={}, bastion_filters={}):
def __init__(self, boto_profile, regions, filters=None, bastion_filters=None):

self.filters = filters
self.filters = filters or []
self.regions = regions.split(',')
self.boto_profile = boto_profile
self.bastion_filters = bastion_filters
self.bastion_filters = bastion_filters or []
self.group_callbacks = []
self.boto3_session = self.create_boto3_session(boto_profile)

# Inventory grouped by instance IDs, tags, security groups, regions,
# and availability zones
Expand All @@ -39,6 +37,25 @@ def __init__(self, boto_profile, regions, filters={}, bastion_filters={}):
# Index of hostname (address) to instance ID
self.index = {}

def create_boto3_session(self, profile_name):
try:
# Use the profile to create a session
session = boto3.Session(profile_name=profile_name)

# Verify region
if not self.regions:
if not session.region_name:
raise NoRegionError
self.regions = [session.region_name]

except NoRegionError:
sys.exit(f"Region not specified and could not be determined for profile: {profile_name}")
except (NoCredentialsError, PartialCredentialsError):
sys.exit(f"Credentials not found or incomplete for profile: {profile_name}")
except Exception as e:
sys.exit(f"An error occurred: {str(e)}")
return session

def get_as_json(self):
self.do_api_calls_update_cache()
return self.json_format_dict(self.inventory, True)
Expand All @@ -55,20 +72,20 @@ def exclude(self, *args):
def group(self, *args):
self.group_callbacks.extend(args)

def find_bastion_box(self, conn):
def find_bastion_box(self, ec2_client):
"""
Find ips for the bastion box
"""

if not self.bastion_filters.values():
if not self.bastion_filters:
return

self.bastion_filters['instance-state-name'] = 'running'
self.bastion_filters.append({'Name': 'instance-state-name', 'Values': ['running']})

for reservation in conn.get_all_instances(
filters=self.bastion_filters):
for instance in reservation.instances:
return instance.ip_address
reservations = ec2_client.describe_instances(Filters=self.bastion_filters)['Reservations']
for reservation in reservations:
for instance in reservation['Instances']:
return instance['PublicIpAddress']

def do_api_calls_update_cache(self):
""" Do API calls to each region, and save data in cache files """
Expand All @@ -80,92 +97,67 @@ def get_instances_by_region(self, region):
"""Makes an AWS EC2 API call to the list of instances in a particular
region
"""
ec2_client = self.boto3_session.client('ec2', region_name=region)

try:
cfg = Config()
cfg.load_credential_file(os.path.expanduser("~/.aws/credentials"))
cfg.load_credential_file(os.path.expanduser("~/.aws/config"))
session_token = cfg.get(self.boto_profile, "aws_session_token")

conn = ec2.connect_to_region(
region,
security_token=session_token,
profile_name=self.boto_profile)

# connect_to_region will fail "silently" by returning None if the
# region name is wrong or not supported
if conn is None:
sys.exit(
"region name: {} likely not supported, or AWS is down. "
"connection to region failed.".format(region))
reservations = ec2_client.describe_instances(Filters=self.filters)['Reservations']

reservations = conn.get_all_instances(filters=self.filters)

bastion_ip = self.find_bastion_box(conn)

instances = []
for reservation in reservations:
instances.extend(reservation.instances)

# sort the instance based on name and index, in this order
def sort_key(instance):
name = instance.tags.get('Name', '')
return "{}-{}".format(name, instance.id)

for instance in sorted(instances, key=sort_key):
self.add_instance(bastion_ip, instance, region)
bastion_ip = self.find_bastion_box(ec2_client)
instances = []
for reservation in reservations:
instances.extend(reservation['Instances'])

except boto.provider.ProfileNotFoundError as e:
raise Exception(
"{}, configure it with 'aws configure --profile {}'".format(e.message, self.boto_profile))
# sort the instance based on name and index, in this order
def sort_key(instance):
name = next((tag['Value'] for tag in instance.get('Tags', [])
if tag['Key'] == 'Name'), '')
return "{}-{}".format(name, instance['InstanceId'])

except boto.exception.BotoServerError as e:
sys.exit(e)
for instance in sorted(instances, key=sort_key):
self.add_instance(bastion_ip, instance, region)

def get_instance(self, region, instance_id):
""" Gets details about a specific instance """
conn = ec2.connect_to_region(region)

ec2_client = self.boto3_session.client('ec2', region_name=region)
# connect_to_region will fail "silently" by returning None if the
# region name is wrong or not supported
if conn is None:
if ec2_client is None:
sys.exit(
"region name: %s likely not supported, or AWS is down. "
"connection to region failed." % region
)

reservations = conn.get_all_instances([instance_id])
reservations = ec2_client.describe_instances(InstanceIds=[instance_id])['Reservations']
for reservation in reservations:
for instance in reservation.instances:
for instance in reservation['Instances']:
return instance

def add_instance(self, bastion_ip, instance, region):
"""
:type instance: boto.ec2.instance.Instance
:type instance: dict
"""

# Only want running instances unless all_instances is True
if instance.state != 'running':
if instance['State']['Name'] != 'running':
return

# Use the instance name instead of the public ip
dest = instance.tags.get('Name', instance.ip_address)
dest = next((tag['Value'] for tag in instance.get('Tags', []) if tag['Key'] == 'Name'), instance.get('PublicIpAddress'))
if not dest:
return

if bastion_ip and bastion_ip != instance.ip_address:
ansible_ssh_host = bastion_ip + "--" + instance.private_ip_address
elif instance.ip_address:
ansible_ssh_host = instance.ip_address
if bastion_ip and bastion_ip != instance.get('PublicIpAddress'):
ansible_ssh_host = bastion_ip + "--" + instance.get('PrivateIpAddress')
elif instance.get('PublicIpAddress'):
ansible_ssh_host = instance.get('PublicIpAddress')
else:
ansible_ssh_host = instance.private_ip_address
ansible_ssh_host = instance.get('PrivateIpAddress')

# Add to index and append the instance id afterwards if it's already
# there
if dest in self.index:
dest = dest + "-" + instance.id.replace("i-", "")
dest = dest + "-" + instance['InstanceId'].replace("i-", "")

self.index[dest] = [region, instance.id]
self.index[dest] = [region, instance['InstanceId']]

# group with dynamic groups
for grouping in set(self.group_callbacks):
Expand All @@ -175,9 +167,9 @@ def add_instance(self, bastion_ip, instance, region):
self.push(self.inventory, group, dest)

# Group by all tags
for tag in instance.tags.values():
if tag:
self.push(self.inventory, tag, dest)
for tag in instance.get('Tags', []):
if tag['Value']:
self.push(self.inventory, tag['Value'], dest)

# Inventory: Group by region
self.push(self.inventory, region, dest)
Expand All @@ -186,56 +178,39 @@ def add_instance(self, bastion_ip, instance, region):
self.push(self.inventory, ansible_ssh_host, dest)

# Inventory: Group by availability zone
self.push(self.inventory, instance.placement, dest)
self.push(self.inventory, instance['Placement']['AvailabilityZone'], dest)

self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(
instance)
self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance)
self.inventory["_meta"]["hostvars"][dest]['ansible_ssh_host'] = ansible_ssh_host

def get_host_info_dict_from_instance(self, instance):
instance_vars = {}
for key in vars(instance):
value = getattr(instance, key)
key = self.to_safe('ec2_' + key)

# Handle complex types
# state/previous_state changed to properties in boto in
# https://github.com/boto/boto/commit/a23c379837f698212252720d2af8dec0325c9518
if key == 'ec2__state':
instance_vars['ec2_state'] = instance.state or ''
instance_vars['ec2_state_code'] = instance.state_code
elif key == 'ec2__previous_state':
instance_vars['ec2_previous_state'] = instance.previous_state or ''
instance_vars['ec2_previous_state_code'] = instance.previous_state_code
elif type(value) in integer_types or isinstance(value, bool):
instance_vars[key] = value
elif type(value) in string_types:
instance_vars[key] = value.strip()
for key, value in instance.items():
safe_key = self.to_safe('ec2_' + key)

if key == 'State':
instance_vars['ec2_state'] = value['Name']
instance_vars['ec2_state_code'] = value['Code']
elif isinstance(value, (int, bool)):
instance_vars[safe_key] = value
elif isinstance(value, str):
instance_vars[safe_key] = value.strip()
elif value is None:
instance_vars[key] = ''
elif key == 'ec2_region':
instance_vars[key] = value.name
elif key == 'ec2__placement':
instance_vars['ec2_placement'] = value.zone
elif key == 'ec2_tags':
for k, v in iteritems(value):
key = self.to_safe('ec2_tag_' + k)
instance_vars[key] = v
elif key == 'ec2_groups':
group_ids = []
group_names = []
for group in value:
group_ids.append(group.id)
group_names.append(group.name)
instance_vars[safe_key] = ''
elif key == 'Placement':
instance_vars['ec2_placement'] = value['AvailabilityZone']
elif key == 'Tags':
for tag in value:
tag_key = self.to_safe('ec2_tag_' + tag['Key'])
instance_vars[tag_key] = tag['Value']
elif key == 'SecurityGroups':
group_ids = [group['GroupId'] for group in value]
group_names = [group['GroupName'] for group in value]
instance_vars["ec2_security_group_ids"] = ','.join(group_ids)
instance_vars["ec2_security_group_names"] = ','.join(
group_names)
# add non ec2 prefix private ip address that are being used in cross provider command
# e.g ssh, sync
instance_vars['private_ip'] = instance_vars.get(
'ec2_private_ip_address', '')
instance_vars['private_ip_address'] = instance_vars.get(
'ec2_private_ip_address', '')
instance_vars["ec2_security_group_names"] = ','.join(group_names)

instance_vars['private_ip'] = instance.get('PrivateIpAddress', '')
instance_vars['private_ip_address'] = instance.get('PrivateIpAddress', '')
return instance_vars

def get_host_info(self):
Expand Down
14 changes: 7 additions & 7 deletions src/ops/inventory/plugin/cns.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,13 @@ def cns(args):
region=region,
boto_profile=profile,
cache=args.get('cache', 3600 * 24),
filters={
'tag:cluster': cns_cluster
},
bastion={
'tag:cluster': cns_cluster,
'tag:role': 'bastion'
}
filters=[
{'Name': 'tag:cluster', 'Values': [cns_cluster]}
],
bastion=[
{'Name': 'tag:cluster', 'Values': [cns_cluster]},
{'Name': 'tag:role', 'Values': ['bastion']}
]
))

merge_inventories(result, json.loads(jsn))
Expand Down
12 changes: 7 additions & 5 deletions src/ops/inventory/plugin/ec2.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,17 @@


def ec2(args):
filters = args.get('filters', {})
bastion_filters = args.get('bastion', {})
filters = args.get('filters', [])
bastion_filters = args.get('bastion', [])

if args.get('cluster') and not args.get('filters'):
filters['tag:cluster'] = args.get('cluster')
filters = [{'Name': 'tag:cluster', 'Values': [args.get('cluster')]}]

if args.get('cluster') and not args.get('bastion'):
bastion_filters['tag:cluster'] = args.get('cluster')
bastion_filters['tag:role'] = 'bastion'
bastion_filters = [
{'Name': 'tag:cluster', 'Values': [args.get('cluster')]},
{'Name': 'tag:role', 'Values': ['bastion']}
]

return Ec2Inventory(boto_profile=args['boto_profile'],
regions=args['region'],
Expand Down

0 comments on commit 1f4826c

Please sign in to comment.