Skip to content

Commit 60d703d

Browse files
committed
switch to irrd graphql API and yaml filter format
1 parent 4963cfb commit 60d703d

File tree

7 files changed

+59
-147
lines changed

7 files changed

+59
-147
lines changed

README.md

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ The output is quite specific to our setup and automation stack, so consider it n
99
## Architecture
1010

1111
The tooling connects to the [Peering-Manager](https://peering-manager.net/) API to fetch BGP sessions.
12-
Then it generates filters for those sessions using the tool bgpq4 that fetches data from an IRR server as data source.
12+
Then it generates filters for those sessions using the IRRD GraphQL API.
1313

1414
We're not only using this to manage peering sessions, but *all* eBGP sessions, so transit and downstream as well.
1515

@@ -117,8 +117,6 @@ If you are using Nix as a package manager you can can start right away using the
117117
For everyone else:
118118
- Install Python3 packages listed in `requirements.txt`
119119
- `pip3 install -r requirements.txt`
120-
- bgpq4
121-
- `apt install bgpq4`
122120

123121

124122
### Usage

flake.nix

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545

4646
devShell = pkgs.mkShell {
4747
buildInputs = with pkgs; [
48-
bgpq4
4948
wanda.dependencyEnv
5049
];
5150
};

nixos-test.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
1313
'';
1414
};
15-
environment.systemPackages = with pkgs; [ wanda wandaTestEnv bgpq4 ];
15+
environment.systemPackages = with pkgs; [ wanda wandaTestEnv ];
1616
};
1717

1818
testScript = { nodes }: let

wanda/as_filter/as_filter.py

Lines changed: 9 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -24,88 +24,26 @@ def prefix_lists(self):
2424
irr_names = self.autos.get_irr_names()
2525

2626
for irr_name in irr_names:
27-
result_entries_v4_cleaned, result_entries_v6_cleaned = self.irrd_client.generate_prefix_lists(irr_name)
27+
result_entries_v4, result_entries_v6 = self.irrd_client.generate_prefix_lists(irr_name)
2828

29-
v4_set.update(result_entries_v4_cleaned)
30-
v6_set.update(result_entries_v6_cleaned)
29+
v4_set.update(result_entries_v4)
30+
v6_set.update(result_entries_v6)
3131

3232
if len(v4_set) == 0 and len(v6_set) == 0 and self.is_customer:
3333
raise Exception(
3434
f"{self.autos} has neither IPv4, nor IPv6 filter lists. Since AS is our customer, we forbid this for security reasons.")
3535

36-
return v4_set, v6_set
36+
return list(v4_set), list(v6_set)
3737

3838
def get_filter_lists(self, enable_extended_filters=False):
3939

4040
irr_names = self.autos.get_irr_names()
41-
file_content = self.irrd_client.generate_input_aspath_access_list(self.autos.asn, irr_names[0])
42-
43-
if file_content is None:
44-
l.warning(f"{self.autos} could not generate as-path access-lists, this breaks configuration syntax..")
45-
return ""
41+
filters = {}
42+
filters['origin_asns'] = self.irrd_client.generate_input_aspath_access_list(self.autos.asn, irr_names[0])
4643

4744
if enable_extended_filters:
4845
v4_set, v6_set = self.prefix_lists
46+
filters[f"v4_prefixes_{self.autos.asn}"] = v4_set
47+
filters[f"v6_prefixes_{self.autos.asn}"] = v6_set
4948

50-
v4_tmpl = ';\n '.join(sorted(v4_set))
51-
v6_tmpl = ';\n '.join(sorted(v6_set))
52-
53-
file_content += f"""
54-
prefix-list AS{self.autos.asn}_V4 {{
55-
{v4_tmpl}
56-
}}
57-
58-
prefix-list AS{self.autos.asn}_V6 {{
59-
{v6_tmpl}
60-
}}
61-
62-
policy-statement POLICY_AS{self.autos.asn}_V4 {{
63-
term FILTER_LISTS {{
64-
from {{
65-
as-path-neighbors as-list AS{self.autos.asn}_NEIGHBOR;
66-
as-path-origins as-list-group AS{self.autos.asn}_ORIGINS;
67-
prefix-list-filter AS{self.autos.asn}_V4 orlonger;
68-
}}
69-
then next policy;
70-
}}
71-
then reject;
72-
}}
73-
74-
policy-statement POLICY_AS{self.autos.asn}_V6 {{
75-
term FILTER_LISTS {{
76-
from {{
77-
as-path-neighbors as-list AS{self.autos.asn}_NEIGHBOR;
78-
as-path-origins as-list-group AS{self.autos.asn}_ORIGINS;
79-
prefix-list-filter AS{self.autos.asn}_V6 orlonger;
80-
}}
81-
then next policy;
82-
}}
83-
then reject;
84-
}}
85-
"""
86-
else:
87-
file_content += f"""
88-
policy-statement POLICY_AS{self.autos.asn}_V4 {{
89-
term IMPORT_AS_PATHS {{
90-
from {{
91-
as-path-neighbors as-list AS{self.autos.asn}_NEIGHBOR;
92-
as-path-origins as-list-group AS{self.autos.asn}_ORIGINS;
93-
}}
94-
then next policy;
95-
}}
96-
then reject;
97-
}}
98-
99-
policy-statement POLICY_AS{self.autos.asn}_V6 {{
100-
term IMPORT_AS_PATHS {{
101-
from {{
102-
as-path-neighbors as-list AS{self.autos.asn}_NEIGHBOR;
103-
as-path-origins as-list-group AS{self.autos.asn}_ORIGINS;
104-
}}
105-
then next policy;
106-
}}
107-
then reject;
108-
}}
109-
"""
110-
111-
return file_content
49+
return filters

wanda/bgp_dg_generation.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ def main_bgp(enlighten_manager, sync_manager, peering_manager_instance, wanda_co
289289

290290
config_hosts = wanda_configuration.get('devices', [])
291291

292-
enabled_routers = list(filter(lambda r: ((not hosts) or (r['hostname'] in hosts)) and ((not config_hosts) or (r['name'] in config_hosts)), routers))
292+
enabled_routers = list(filter(lambda r: ((not hosts) or (r['hostname'] in hosts)) and ((not config_hosts) or (r['hostname'] in config_hosts)), routers))
293293
e_routers = enlighten_manager.counter(total=len(enabled_routers), desc='Generating Configurations', unit='Router')
294294

295295
for router in enabled_routers:

wanda/filter_list_generation.py

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import platform
33
from collections import defaultdict
44
from multiprocessing.pool import Pool
5+
import yaml
56

67
from wanda.as_filter.as_filter import ASFilter
78
from wanda.autonomous_system.autonomous_system import AutonomousSystem
@@ -65,11 +66,12 @@ def main_customer_filter_lists(
6566
router_per_as = {}
6667
enabled_asn = set()
6768
extended_filtering_as = set()
69+
config_hosts = wanda_configuration.get('devices', [])
6870

6971
for dp in dp_list:
7072
router_hostname = dp['router']['hostname']
7173

72-
if hosts and router_hostname not in hosts:
74+
if (hosts and router_hostname not in hosts) or router_hostname not in config_hosts:
7375
continue
7476

7577
asn = dp['autonomous_system']['asn']
@@ -96,12 +98,12 @@ def main_customer_filter_lists(
9698
asn = ixp['autonomous_system']['asn']
9799
router_hostname = connection['router']['hostname']
98100

99-
if hosts and router_hostname not in hosts:
101+
if (hosts and router_hostname not in hosts) or router_hostname not in config_hosts:
100102
continue
101103

102104
if ixp['is_route_server']:
103105
if router_hostname not in router_per_as:
104-
router_per_as[router_hostname] = {}
106+
router_per_as[router_hostname] = set()
105107
continue
106108

107109
enabled_asn.add(asn)
@@ -159,14 +161,13 @@ def main_customer_filter_lists(
159161
l.info(f"Skipping {router['hostname']}, because there is no 'automated' tag. ")
160162
continue
161163

162-
config_parts = []
164+
config_parts = {}
163165

164166
for asn in as_list:
165167
if asn in filter_lists:
166-
config_parts.append(filter_lists[asn])
168+
config_parts[f"AS{asn}"] = filter_lists[asn]
167169

168-
pathlib.Path("./generated_vars").mkdir(parents=True, exist_ok=True)
169-
with open('./generated_vars/filter_groups-' + router_hostname + '.tmpl', 'w') as yaml_file:
170-
yaml_file.write("\n".join(config_parts))
170+
with open('./machines/' + router_hostname.split(".")[0] + '/generated-wanda-filters.yaml', 'w') as yaml_file:
171+
yaml_file.write(yaml.safe_dump(config_parts))
171172

172173
return 0

wanda/irrd_client.py

Lines changed: 38 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
import subprocess
1+
import requests
2+
import json
23
import re
34

45
from wanda.logger import Logger
@@ -8,74 +9,49 @@
89

910
class IRRDClient:
1011

11-
POSSIBLE_RETRIES = 3
12-
1312
def __init__(self, irrd_url):
14-
self.data = []
15-
self.irrdURL = irrd_url
16-
self.host_params = ['-h', irrd_url]
17-
18-
def call_subprocess(self, command_array):
19-
20-
current_try = 1
21-
22-
while current_try <= self.POSSIBLE_RETRIES:
23-
result = subprocess.run(command_array, capture_output=True)
24-
if result.returncode == 0:
25-
result_str = result.stdout.decode("utf-8")
26-
return result_str
13+
self.irrdURL = f"https://{irrd_url}/graphql/"
2714

28-
current_try += 1
15+
def fetch_graphql_data(self, query):
16+
response = requests.post(url=self.irrdURL, json={"query": query})
17+
if response.status_code == 200:
18+
return json.loads(response.content)["data"]
2919

30-
l.error(f"Failed to execute command: {' '.join(command_array)}")
31-
raise Exception("bgpq4 could not be called successfully, this may be an programming error or a bad internet connection.")
32-
33-
def call_bgpq4_aspath_access_list(self, asn, irr_name):
34-
command_array = ["bgpq4", *self.host_params, "-H", str(asn), "-W 100", "-J", "-l", f"AS{asn}_ORIGINS", irr_name]
35-
return self.call_subprocess(command_array)
20+
return None
3621

3722
def generate_input_aspath_access_list(self, asn, irr_name):
38-
# bgpq4 AS-TELIANET-V6 -H 1299 -W 100 -J -l AS1299_ORIGINS
39-
result_str = self.call_bgpq4_aspath_access_list(asn, irr_name)
40-
m = re.search(r'.*as-list-group.*{(.|\n)*?}', result_str)
41-
42-
if m:
43-
# Technically, only adding the AS..._NEIGHBOR list would work, but we do some cleaning for better quality of the generated configuration
44-
45-
lines = m[0].split("\n")
46-
new_lines = list()
47-
new_lines.append(f"as-list AS{asn}_NEIGHBOR members {asn};")
48-
indent_count = 0
23+
if re.match(r"^AS\d+$", irr_name):
24+
return [ irr_name[2:] ]
4925

50-
for line in lines:
51-
line_without_prefixed_spaces = line.lstrip()
26+
body = f"""
27+
{{
28+
recursiveSetMembers(setNames: ["{irr_name}"], depth: 5) {{ members }}
29+
}}
30+
"""
31+
result = self.fetch_graphql_data(body)
5232

53-
if '}' in line_without_prefixed_spaces:
54-
indent_count -= 1
33+
# return unique members that are ASNs
34+
members = set(result["recursiveSetMembers"][0]["members"])
35+
return [int(i[2:]) for i in members if re.match(r"^AS\d+$", i)]
5536

56-
spaces = [" " for _ in range(indent_count * 4)]
57-
new_lines.append("".join(spaces) + line_without_prefixed_spaces)
58-
59-
if '{' in line_without_prefixed_spaces:
60-
indent_count += 1
61-
62-
return "\n".join(new_lines)
63-
64-
return None
65-
66-
def call_bgpq4_prefix_lists(self, irr_name, ip_version):
67-
command_array = ["bgpq4", *self.host_params, f"-{ip_version}", "-F", "%n/%l\n", irr_name]
68-
return self.call_subprocess(command_array)
6937

7038
def generate_prefix_lists(self, irr_name):
71-
result_v4 = self.call_bgpq4_prefix_lists(irr_name, 4)
72-
result_v6 = self.call_bgpq4_prefix_lists(irr_name, 6)
73-
74-
result_entries_v4 = result_v4.splitlines()
75-
result_entries_v6 = result_v6.splitlines()
76-
77-
# Stripping empty lines
78-
result_entries_v4_cleaned = [x for x in result_entries_v4 if x]
79-
result_entries_v6_cleaned = [x for x in result_entries_v6 if x]
80-
81-
return result_entries_v4_cleaned, result_entries_v6_cleaned
39+
if re.match(r"^AS\d+$", irr_name):
40+
# ASNs
41+
body = f"""
42+
{{
43+
v4: asnPrefixes(asns: ["{irr_name[2:]}"], ipVersion: 4) {{ prefixes }}
44+
v6: asnPrefixes(asns: ["{irr_name[2:]}"], ipVersion: 6) {{ prefixes }}
45+
}}
46+
"""
47+
else:
48+
# AS-Set
49+
body = f"""
50+
{{
51+
v4: asSetPrefixes(setNames: ["{irr_name}"], ipVersion: 4) {{ prefixes }}
52+
v6: asSetPrefixes(setNames: ["{irr_name}"], ipVersion: 6) {{ prefixes }}
53+
}}
54+
"""
55+
result = self.fetch_graphql_data(body)
56+
57+
return result["v4"][0]["prefixes"], result["v6"][0]["prefixes"]

0 commit comments

Comments
 (0)