simln: export data inside docker compose network

Author: pinheadmz

src/cli/network.py

@@ -143,8 +143,9 @@ def connected(network: str):
 
 @network.command()
 @click.option("--network", default="warnet", show_default=True)
-def export(network):
+def export(network: str):
     """
-    Export all [network] data for sim-ln to subdirectory
+    Export all [network] data for a "simln" service running in a container
+    on the network. Returns True on success.
     """
     print(rpc_call("network_export", {"network": network}))

src/warnet/lnnode.py

@@ -1,4 +1,5 @@
-import os
+import io
+import tarfile
 
 from backends import BackendInterface, ServiceType
 from warnet.utils import exponential_backoff, generate_ipv4_addr, handle_json
@@ -115,30 +116,38 @@ def generate_cli_command(self, command: list[str]):
                 raise Exception(f"Unsupported LN implementation: {self.impl}")
         return cmd
 
-    def export(self, config, subdir):
-        container_name = self.backend.get_container_name(self.tank.index, ServiceType.LIGHTNING)
-        macaroon_filename = f"{container_name}_admin.macaroon"
-        cert_filename = f"{container_name}_tls.cert"
-        macaroon_path = os.path.join(subdir, macaroon_filename)
-        cert_path = os.path.join(subdir, cert_filename)
+    def export(self, config: object, tar_file):
+        # Retrieve the credentials
         macaroon = self.backend.get_file(
             self.tank.index,
             ServiceType.LIGHTNING,
             "/root/.lnd/data/chain/bitcoin/regtest/admin.macaroon",
         )
-        cert = self.backend.get_file(self.tank.index, ServiceType.LIGHTNING, "/root/.lnd/tls.cert")
+        cert = self.backend.get_file(
+            self.tank.index,
+            ServiceType.LIGHTNING,
+            "/root/.lnd/tls.cert"
+        )
 
-        with open(macaroon_path, "wb") as f:
-            f.write(macaroon)
+        # Add the files to the in-memory tar archive
+        container_name = self.backend.get_container_name(self.tank.index, ServiceType.LIGHTNING)
+        macaroon_filename = f"{container_name}_admin.macaroon"
+        cert_filename = f"{container_name}_tls.cert"
 
-        with open(cert_path, "wb") as f:
-            f.write(cert)
+        tarinfo1 = tarfile.TarInfo(name=macaroon_filename)
+        tarinfo1.size = len(macaroon)
+        fileobj1 = io.BytesIO(macaroon)
+        tar_file.addfile(tarinfo=tarinfo1, fileobj=fileobj1)
+        tarinfo2 = tarfile.TarInfo(name=cert_filename)
+        tarinfo2.size = len(cert)
+        fileobj2 = io.BytesIO(cert)
+        tar_file.addfile(tarinfo=tarinfo2, fileobj=fileobj2)
 
         config["nodes"].append(
             {
                 "id": container_name,
-                "address": f"https://{self.ipv4}:{self.rpc_port}",
-                "macaroon": macaroon_path,
-                "cert": cert_path,
+                "address": f"https://{container_name}:{self.rpc_port}",
+                "macaroon": f"/simln/{macaroon_filename}",
+                "cert": f"/simln/{cert_filename}",
             }
         )

src/warnet/server.py

@@ -1,5 +1,6 @@
 import argparse
 import base64
+import io
 import json
 import logging
 import logging.config
@@ -10,6 +11,7 @@
 import signal
 import subprocess
 import sys
+import tarfile
 import tempfile
 import threading
 import time
@@ -268,20 +270,32 @@ def tank_messages(self, network: str, node_a: int, node_b: int) -> str:
             self.logger.error(msg)
             raise ServerError(message=msg) from e
 
-    def network_export(self, network: str) -> str:
+    def network_export(self, network: str) -> bool:
         """
-        Export all data for sim-ln to subdirectory
+        Export all data for a simln container running on the network
         """
-        try:
-            wn = self.get_warnet(network)
-            subdir = os.path.join(wn.config_dir, "simln")
-            os.makedirs(subdir, exist_ok=True)
-            wn.export(subdir)
-            return subdir
-        except Exception as e:
-            msg = f"Error exporting network: {e}"
-            self.logger.error(msg)
-            raise ServerError(message=msg) from e
+        wn = self.get_warnet(network)
+
+        # JSON object that will eventually be written to simln config file
+        config = {"nodes": []}
+        # In-memory file to build tar archive
+        tar_buffer = io.BytesIO()
+        with tarfile.open(fileobj=tar_buffer, mode="w") as tar_file:
+            # tank LN nodes add their credentials to tar archive
+            wn.export(config, tar_file)
+            # write config file
+            config_bytes = json.dumps(config).encode('utf-8')
+            config_stream = io.BytesIO(config_bytes)
+            tarinfo = tarfile.TarInfo(name="sim.json")
+            tarinfo.size = len(config_bytes)
+            tar_file.addfile(tarinfo=tarinfo, fileobj=config_stream)
+
+        with open(wn.config_dir / "simln.tar", "wb") as output:
+            tar_buffer.seek(0)
+            output.write(tar_buffer.read())
+
+        subprocess.run(["tar", "-xf", wn.config_dir / 'simln.tar', "-C", wn.config_dir / 'simln'])
+        return True
 
     def scenarios_available(self) -> list[tuple]:
         """

src/warnet/services.py

@@ -83,6 +83,10 @@
         "backends": ["compose", "k8s"],
         "image": "bitcoindevproject/simln:0.2.0",
         "container_name_suffix": "simln",
-        "args": ["--log-level=debug"]
+        "environment": [
+            "LOG_LEVEL=debug",
+            "SIMFILE_PATH=/simln/sim.json"
+        ],
+        "config_files": ["/simln:/simln"]
     },
 }

src/warnet/tank.py

@@ -169,6 +169,6 @@ def apply_network_conditions(self):
                 f"Error applying network conditions to tank {self.index}: `{self.netem}` ({e})"
             )
 
-    def export(self, config, subdir):
+    def export(self, config: object, tar_file):
         if self.lnnode is not None:
-            self.lnnode.export(config, subdir)
+            self.lnnode.export(config, tar_file)

src/warnet/warnet.py

@@ -5,7 +5,6 @@
 import base64
 import json
 import logging
-import os
 import shutil
 from pathlib import Path
 
@@ -136,6 +135,8 @@ def from_network(cls, network_name, backend="compose"):
         self.graph = networkx.read_graphml(Path(self.config_dir / self.graph_name), node_type=int, force_multigraph=True)
         validate_graph_schema(self.graph)
         self.tanks_from_graph()
+        if "services" in self.graph.graph:
+            self.services = self.graph.graph["services"].split()
         for tank in self.tanks:
             tank._ipv4 = self.container_interface.get_tank_ipv4(tank.index)
         return self
@@ -237,15 +238,9 @@ def write_prometheus_config(self):
         except Exception as e:
             logger.error(f"An error occurred while writing to {prometheus_path}: {e}")
 
-    def export(self, subdir):
-        if self.backend != "compose":
-            raise NotImplementedError("Export is only supported for compose backend")
-        config = {"nodes": []}
+    def export(self, config: object, tar_file):
         for tank in self.tanks:
-            tank.export(config, subdir)
-        config_path = os.path.join(subdir, "sim.json")
-        with open(config_path, "a") as f:
-            json.dump(config, f)
+            tank.export(config, tar_file)
 
     def wait_for_health(self):
         self.container_interface.wait_for_healthy_tanks(self)

test/data/ln.graphml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?><graphml xmlns="http://graphml.graphdrawing.org/xmlns">
   <key id="version"         attr.name="version"          attr.type="string"   for="node" />
+  <key id="services"        attr.name="services"         attr.type="string"   for="graph" />
   <key id="image"           attr.name="image"            attr.type="string"   for="node" />
   <key id="bitcoin_config"  attr.name="bitcoin_config"   attr.type="string"   for="node" />
   <key id="tc_netem"        attr.name="tc_netem"         attr.type="string"   for="node" />
@@ -14,6 +15,7 @@
   <key id="source_policy"   attr.name="source_policy"    attr.type="string"   for="edge" />
   <key id="target_policy"   attr.name="target_policy"    attr.type="string"   for="edge" />
   <graph edgedefault="directed">
+    <data key="services">simln</data>
     <node id="0">
         <data key="version">26.0</data>
         <data key="bitcoin_config">-uacomment=w0</data>