Skip to content

Commit 892553a

Browse files
authored
Merge pull request #288 from pinheadmz/circuitbreaker
2 parents ca08973 + dc1d913 commit 892553a

File tree

11 files changed

+139
-24
lines changed

11 files changed

+139
-24
lines changed

src/backends/backend_interface.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
class ServiceType(Enum):
77
BITCOIN = 1
88
LIGHTNING = 2
9+
CIRCUITBREAKER = 3
910

1011

1112
class BackendInterface(ABC):

src/backends/compose/compose_backend.py

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
GRAFANA_PROVISIONING = "grafana-provisioning"
4040
CONTAINER_PREFIX_BITCOIND = "tank-bitcoin"
4141
CONTAINER_PREFIX_LN = "tank-ln"
42+
CONTAINER_PREFIX_CIRCUITBREAKER = "tank-ln-cb"
43+
LND_MOUNT_PATH = '/root/.lnd'
4244

4345
logger = logging.getLogger("docker-interface")
4446
logging.getLogger("docker.utils.config").setLevel(logging.WARNING)
@@ -91,15 +93,15 @@ def up(self, *_):
9193
)
9294

9395
def down(self, warnet):
94-
command = ["docker", "compose", "down"]
96+
command = ["docker", "compose", "down", "-v"]
9597
try:
9698
with subprocess.Popen(
9799
command,
98100
cwd=str(self.config_dir),
99101
stdout=subprocess.PIPE,
100102
stderr=subprocess.STDOUT,
101103
) as process:
102-
logger.debug(f"Running docker compose down with PID {process.pid}")
104+
logger.debug(f"Running 'docker compose down -v' with PID {process.pid}")
103105
if process.stdout:
104106
for line in process.stdout:
105107
logger.info(line.decode().rstrip())
@@ -114,6 +116,8 @@ def get_container_name(self, tank_index: int, service: ServiceType) -> str:
114116
return f"{self.network_name}-{CONTAINER_PREFIX_BITCOIND}-{tank_index:06}"
115117
case ServiceType.LIGHTNING:
116118
return f"{self.network_name}-{CONTAINER_PREFIX_LN}-{tank_index:06}"
119+
case ServiceType.CIRCUITBREAKER:
120+
return f"{self.network_name}-{CONTAINER_PREFIX_CIRCUITBREAKER}-{tank_index:06}"
117121
case _:
118122
raise Exception("Unsupported service type")
119123

@@ -301,7 +305,7 @@ def _write_docker_compose(self, warnet):
301305

302306
# Pass services object to each tank so they can add whatever they need.
303307
for tank in warnet.tanks:
304-
self.add_services(tank, compose["services"])
308+
self.add_services(tank, compose)
305309

306310
# Initialize services and add them to the compose
307311
services = [
@@ -362,7 +366,8 @@ def copy_configs(self, tank):
362366
shutil.copyfile(TEMPLATES / ENTRYPOINT_NAME, tank.config_dir / ENTRYPOINT_NAME)
363367
set_execute_permission(tank.config_dir / ENTRYPOINT_NAME)
364368

365-
def add_services(self, tank: Tank, services):
369+
def add_services(self, tank: Tank, compose):
370+
services = compose["services"]
366371
assert tank.index is not None
367372
container_name = self.get_container_name(tank.index, ServiceType.BITCOIN)
368373
services[container_name] = {}
@@ -411,7 +416,7 @@ def add_services(self, tank: Tank, services):
411416
services[container_name]["labels"].update({"collect_logs": True})
412417

413418
if tank.lnnode is not None:
414-
self.add_lnd_service(tank, services)
419+
self.add_lnd_service(tank, compose)
415420

416421
# Add the prometheus data exporter in a neighboring container
417422
if tank.exporter:
@@ -427,8 +432,10 @@ def add_services(self, tank: Tank, services):
427432
"networks": [tank.network_name],
428433
}
429434

430-
def add_lnd_service(self, tank, services):
435+
def add_lnd_service(self, tank, compose):
436+
services = compose["services"]
431437
ln_container_name = self.get_container_name(tank.index, ServiceType.LIGHTNING)
438+
ln_cb_container_name = self.get_container_name(tank.index, ServiceType.CIRCUITBREAKER)
432439
bitcoin_container_name = self.get_container_name(tank.index, ServiceType.BITCOIN)
433440
# These args are appended to the Dockerfile `ENTRYPOINT ["lnd"]`
434441
args = [
@@ -447,6 +454,7 @@ def add_lnd_service(self, tank, services):
447454
f"--externalip={tank.lnnode.ipv4}",
448455
f"--rpclisten=0.0.0.0:{tank.lnnode.rpc_port}",
449456
f"--alias={tank.index}",
457+
f"--tlsextradomain={ln_container_name}",
450458
]
451459
services[ln_container_name] = {
452460
"container_name": ln_container_name,
@@ -477,6 +485,23 @@ def add_lnd_service(self, tank, services):
477485
)
478486
if tank.collect_logs:
479487
services[ln_container_name]["labels"].update({"collect_logs": True})
488+
if tank.lnnode.cb is not None:
489+
services[ln_container_name].update({
490+
"volumes": [f"{ln_container_name}-data:{LND_MOUNT_PATH}"]
491+
})
492+
services[ln_cb_container_name] = {
493+
"container_name": ln_cb_container_name,
494+
"image": tank.lnnode.cb,
495+
"volumes": [f"{ln_container_name}-data:{LND_MOUNT_PATH}"],
496+
"command": "--network=regtest " +
497+
f"--rpcserver={ln_container_name}:{tank.lnnode.rpc_port} " +
498+
f" --tlscertpath={LND_MOUNT_PATH}/tls.cert " +
499+
f" --macaroonpath={LND_MOUNT_PATH}/data/chain/bitcoin/regtest/admin.macaroon",
500+
"networks": [tank.network_name],
501+
"restart": "on-failure",
502+
}
503+
compose["volumes"].update({f"{ln_container_name}-data": None})
504+
480505

481506
def get_ipv4_address(self, container: Container) -> str:
482507
"""

src/backends/kubernetes/kubernetes_backend.py

Lines changed: 58 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@
2323
POD_PREFIX = "tank"
2424
BITCOIN_CONTAINER_NAME = "bitcoin"
2525
LN_CONTAINER_NAME = "ln"
26+
LN_CB_CONTAINER_NAME = "ln-cb"
2627
MAIN_NAMESPACE = "warnet"
2728
PROMETHEUS_METRICS_PORT = 9332
29+
LND_MOUNT_PATH = '/root/.lnd'
2830

2931

3032
logger = logging.getLogger("KubernetesBackend")
@@ -107,7 +109,7 @@ def get_file(self, tank_index: int, service: ServiceType, file_path: str):
107109
return decoded_bytes
108110

109111
def get_pod_name(self, tank_index: int, type: ServiceType) -> str:
110-
if type == ServiceType.LIGHTNING:
112+
if type == ServiceType.LIGHTNING or type == ServiceType.CIRCUITBREAKER:
111113
return f"{self.network_name}-{POD_PREFIX}-ln-{tank_index:06d}"
112114
return f"{self.network_name}-{POD_PREFIX}-{tank_index:06d}"
113115

@@ -173,18 +175,19 @@ def get_status(self, tank_index: int, service: ServiceType) -> RunningStatus:
173175

174176
def exec_run(self, tank_index: int, service: ServiceType, cmd: str):
175177
pod_name = self.get_pod_name(tank_index, service)
176-
if service == ServiceType.BITCOIN:
177-
exec_cmd = ["/bin/bash", "-c", f"{cmd}"]
178-
elif service == ServiceType.LIGHTNING:
179-
exec_cmd = ["/bin/sh", "-c", f"{cmd}"]
178+
exec_cmd = ["/bin/sh", "-c", f"{cmd}"]
180179
self.log.debug(f"Running {exec_cmd=:} on {tank_index=:}")
180+
if service == ServiceType.BITCOIN:
181+
container = BITCOIN_CONTAINER_NAME
182+
if service == ServiceType.LIGHTNING:
183+
container = LN_CONTAINER_NAME
184+
if service == ServiceType.CIRCUITBREAKER:
185+
container = LN_CB_CONTAINER_NAME
181186
result = stream(
182187
self.client.connect_get_namespaced_pod_exec,
183188
pod_name,
184189
self.namespace,
185-
container=BITCOIN_CONTAINER_NAME
186-
if service == ServiceType.BITCOIN
187-
else LN_CONTAINER_NAME,
190+
container=container,
188191
command=exec_cmd,
189192
stderr=True,
190193
stdin=False,
@@ -432,7 +435,7 @@ def remove_prometheus_service_monitors(self, tanks):
432435
if e.status != 404:
433436
raise e
434437

435-
def create_lnd_container(self, tank, bitcoind_service_name) -> client.V1Container:
438+
def create_lnd_container(self, tank, bitcoind_service_name, volume_mounts) -> client.V1Container:
436439
# These args are appended to the Dockerfile `ENTRYPOINT ["lnd"]`
437440
bitcoind_rpc_host = f"{bitcoind_service_name}.{self.namespace}"
438441
lightning_dns = f"lightning-{tank.index}.{self.namespace}"
@@ -475,12 +478,33 @@ def create_lnd_container(self, tank, bitcoind_service_name) -> client.V1Containe
475478
privileged=True,
476479
capabilities=client.V1Capabilities(add=["NET_ADMIN", "NET_RAW"]),
477480
),
481+
volume_mounts=volume_mounts,
478482
)
479483
self.log.debug(f"Created lightning container for tank {tank.index}")
480484
return lightning_container
481485

486+
def create_circuitbreaker_container(self, tank, volume_mounts) -> client.V1Container:
487+
self.log.debug(f"Creating circuitbreaker container for tank {tank.index}")
488+
cb_container = client.V1Container(
489+
name=LN_CB_CONTAINER_NAME,
490+
image=tank.lnnode.cb,
491+
args=[
492+
"--network=regtest",
493+
f"--rpcserver=localhost:{tank.lnnode.rpc_port}",
494+
f"--tlscertpath={LND_MOUNT_PATH}/tls.cert",
495+
f"--macaroonpath={LND_MOUNT_PATH}/data/chain/bitcoin/regtest/admin.macaroon"
496+
],
497+
security_context=client.V1SecurityContext(
498+
privileged=True,
499+
capabilities=client.V1Capabilities(add=["NET_ADMIN", "NET_RAW"]),
500+
),
501+
volume_mounts=volume_mounts,
502+
)
503+
self.log.debug(f"Created circuitbreaker container for tank {tank.index}")
504+
return cb_container
505+
482506
def create_pod_object(
483-
self, tank: Tank, container: client.V1Container, name: str
507+
self, tank: Tank, containers: list[client.V1Container], volumes: list[client.V1Volume], name: str
484508
) -> client.V1Pod:
485509
# Create and return a Pod object
486510
# TODO: pass a custom namespace , e.g. different warnet sims can be deployed into diff namespaces
@@ -500,7 +524,8 @@ def create_pod_object(
500524
# Might need some more thinking on the pod restart policy, setting to Never for now
501525
# This means if a node has a problem it dies
502526
restart_policy="OnFailure",
503-
containers=[container],
527+
containers=containers,
528+
volumes=volumes,
504529
),
505530
)
506531

@@ -588,7 +613,7 @@ def deploy_pods(self, warnet):
588613
# Create and deploy bitcoind pod and service
589614
bitcoind_container = self.create_bitcoind_container(tank)
590615
bitcoind_pod = self.create_pod_object(
591-
tank, bitcoind_container, self.get_pod_name(tank.index, ServiceType.BITCOIN)
616+
tank, [bitcoind_container], [], self.get_pod_name(tank.index, ServiceType.BITCOIN)
592617
)
593618

594619
if tank.exporter and self.check_logging_crds_installed():
@@ -609,11 +634,30 @@ def deploy_pods(self, warnet):
609634

610635
# Create and deploy LND pod
611636
if tank.lnnode:
612-
lnd_container = self.create_lnd_container(tank, bitcoind_service.metadata.name)
637+
conts = []
638+
vols = []
639+
volume_mounts = []
640+
if tank.lnnode.cb:
641+
# Create a shared volume between containers in the pod
642+
volume_name = f"ln-cb-data-{tank.index}"
643+
vols.append(client.V1Volume(
644+
name=volume_name,
645+
empty_dir=client.V1EmptyDirVolumeSource()
646+
))
647+
volume_mounts.append(client.V1VolumeMount(
648+
name=volume_name,
649+
mount_path=LND_MOUNT_PATH,
650+
))
651+
# Add circuit breaker container
652+
conts.append(self.create_circuitbreaker_container(tank, volume_mounts))
653+
# Add lnd container
654+
conts.append(self.create_lnd_container(tank, bitcoind_service.metadata.name, volume_mounts))
655+
# Put it all together in a pod
613656
lnd_pod = self.create_pod_object(
614-
tank, lnd_container, self.get_pod_name(tank.index, ServiceType.LIGHTNING)
657+
tank, conts, vols, self.get_pod_name(tank.index, ServiceType.LIGHTNING)
615658
)
616659
self.client.create_namespaced_pod(namespace=self.namespace, body=lnd_pod)
660+
# Create service for the pod
617661
lightning_service = self.create_lightning_service(tank)
618662
try:
619663
self.client.delete_namespaced_service(

src/cli/network.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,9 +122,12 @@ def status(network: str):
122122
assert isinstance(result, list), "Result is not a list" # Make mypy happy
123123
for tank in result:
124124
lightning_status = ""
125+
circuitbreaker_status = ""
125126
if "lightning_status" in tank:
126127
lightning_status = f"\tLightning: {tank['lightning_status']}"
127-
print(f"Tank: {tank['tank_index']} \tBitcoin: {tank['bitcoin_status']}{lightning_status}")
128+
if "circuitbreaker_status" in tank:
129+
circuitbreaker_status = f"\tCircuit Breaker: {tank['circuitbreaker_status']}"
130+
print(f"Tank: {tank['tank_index']} \tBitcoin: {tank['bitcoin_status']}{lightning_status}{circuitbreaker_status}")
128131

129132

130133
@network.command()

src/templates/node_schema.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@
1212
"collect_logs": {"type": "boolean", "default": false},
1313
"build_args": {"type": "string"},
1414
"ln": {"type": "string"},
15-
"ln-image": {"type": "string"}
15+
"ln-image": {"type": "string"},
16+
"ln-cb-image": {"type": "string"}
1617
},
1718
"additionalProperties": false,
1819
"oneOf": [

src/warnet/lnnode.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88

99
class LNNode:
10-
def __init__(self, warnet, tank, impl, image, backend: BackendInterface):
10+
def __init__(self, warnet, tank, impl, image, backend: BackendInterface, cb=None):
1111
self.warnet = warnet
1212
self.tank = tank
1313
assert impl == "lnd"
@@ -16,6 +16,7 @@ def __init__(self, warnet, tank, impl, image, backend: BackendInterface):
1616
if image:
1717
self.image = image
1818
self.backend = backend
19+
self.cb = cb
1920
self.ipv4 = generate_ipv4_addr(self.warnet.subnet)
2021
self.rpc_port = 10009
2122

@@ -26,6 +27,12 @@ def __str__(self):
2627
def status(self) -> RunningStatus:
2728
return self.warnet.container_interface.get_status(self.tank.index, ServiceType.LIGHTNING)
2829

30+
@property
31+
def cb_status(self) -> RunningStatus:
32+
if not self.cb:
33+
return None
34+
return self.warnet.container_interface.get_status(self.tank.index, ServiceType.CIRCUITBREAKER)
35+
2936
@exponential_backoff(max_retries=20, max_delay=300)
3037
@handle_json
3138
def lncli(self, cmd) -> dict:

src/warnet/server.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import jsonschema
2222
import networkx as nx
2323
import scenarios
24+
from backends import ServiceType
2425
from flask import Flask, jsonify, request
2526
from flask_jsonrpc.app import JSONRPC
2627
from flask_jsonrpc.exceptions import ServerError
@@ -180,6 +181,7 @@ def setup_rpc(self):
180181
self.jsonrpc.register(self.graph_validate)
181182
# Debug
182183
self.jsonrpc.register(self.generate_deployment)
184+
self.jsonrpc.register(self.exec_run)
183185
# Server
184186
self.jsonrpc.register(self.server_stop)
185187
# Logs
@@ -563,6 +565,8 @@ def network_status(self, network: str = "warnet") -> list[dict]:
563565
status = {"tank_index": tank.index, "bitcoin_status": tank.status.name.lower()}
564566
if tank.lnnode is not None:
565567
status["lightning_status"] = tank.lnnode.status.name.lower()
568+
if tank.lnnode.cb is not None:
569+
status["circuitbreaker_status"] = tank.lnnode.cb_status.name.lower()
566570
stats.append(status)
567571
return stats
568572
except Exception as e:
@@ -612,6 +616,14 @@ def logs_grep(self, pattern: str, network: str = "warnet") -> str:
612616
self.logger.error(msg)
613617
raise ServerError(message=msg) from e
614618

619+
def exec_run(self, index: int, service_type: int, cmd: str, network: str = "warnet") -> str:
620+
"""
621+
Execute an arbitrary command in an arbitrary container,
622+
identified by tank index and ServiceType
623+
"""
624+
wn = Warnet.from_network(network, self.backend)
625+
return wn.container_interface.exec_run(index, ServiceType(service_type), cmd)
626+
615627

616628
def run_server():
617629
parser = argparse.ArgumentParser(description="Run the server")

src/warnet/tank.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,8 @@ def parse_graph_node(self, node):
7575
if "ln" in node:
7676
impl = node["ln"]
7777
image = node.get("ln-image", None)
78-
self.lnnode = LNNode(self.warnet, self, impl, image, self.warnet.container_interface)
78+
cb_image = node.get("ln-cb-image", None)
79+
self.lnnode = LNNode(self.warnet, self, impl, image, self.warnet.container_interface, cb_image)
7980

8081
self.config_dir = self.warnet.config_dir / str(self.suffix)
8182
self.config_dir.mkdir(parents=True, exist_ok=True)

test/data/ln.graphml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
<key attr.name="tc_netem" attr.type="string" for="node" id="tc_netem"/>
55
<key attr.name="ln" attr.type="string" for="node" id="ln"/>
66
<key attr.name="ln-image" attr.type="string" for="node" id="ln-image"/>
7+
<key attr.name="ln-cb-image" attr.type="string" for="node" id="ln-cb-image"/>
78
<key attr.name="channel" attr.type="string" for="edge" id="channel"/>
89
<key attr.name="collect_logs" attr.type="boolean" for="node" id="collect_logs"/>
910
<key attr.name="image" attr.type="string" for="node" id="image"/>
@@ -19,12 +20,14 @@
1920
<data key="bitcoin_config">-uacomment=w1</data>
2021
<data key="ln">lnd</data>
2122
<data key="ln-image">lightninglabs/lnd:v0.15.5-beta</data>
23+
<data key="ln-cb-image">pinheadmz/circuitbreaker:278737d</data>
2224
<data key="collect_logs">true</data>
2325
</node>
2426
<node id="2">
2527
<data key="version">26.0</data>
2628
<data key="bitcoin_config">-uacomment=w2</data>
2729
<data key="ln">lnd</data>
30+
<data key="ln-cb-image">pinheadmz/circuitbreaker:278737d</data>
2831
</node>
2932
<node id="3">
3033
<data key="version">26.0</data>

0 commit comments

Comments
 (0)