Merge pull request #315 from pinheadmz/ln-ch-policies

support channel policies in LN edges

Author: pinheadmz

Diff for: docs/graph.md

@@ -52,9 +52,12 @@ lightning network channel (see [lightning.md](lightning.md)).
   <key id="collect_logs"    attr.name="collect_logs"     attr.type="boolean"  for="node" />
   <key id="build_args"      attr.name="build_args"       attr.type="string"   for="node" />
   <key id="ln"              attr.name="ln"               attr.type="string"   for="node" />
-  <key id="ln-image"        attr.name="ln-image"         attr.type="string"   for="node" />
-  <key id="ln-cb-image"     attr.name="ln-cb-image"      attr.type="string"   for="node" />
-  <key id="channel"         attr.name="channel"          attr.type="number"   for="edge" />
+  <key id="ln_image"        attr.name="ln_image"         attr.type="string"   for="node" />
+  <key id="ln_cb_image"     attr.name="ln_cb_image"      attr.type="string"   for="node" />
+  <key id="ln_config"       attr.name="ln_config"        attr.type="string"   for="node" />
+  <key id="channel_open"    attr.name="channel_open"     attr.type="string"   for="edge" />
+  <key id="source_policy"   attr.name="source_policy"    attr.type="string"   for="edge" />
+  <key id="target_policy"   attr.name="target_policy"    attr.type="string"   for="edge" />
   <graph edgedefault="directed">
     <!-- <nodes> -->
     <!-- <edges> -->
@@ -72,6 +75,9 @@ lightning network channel (see [lightning.md](lightning.md)).
 | collect_logs   | node  | boolean | False     | Whether to collect Bitcoin Core debug logs with Promtail                                                                                                            |
 | build_args     | node  | string  |           | A string of configure options used when building Bitcoin Core from source code, e.g. '--without-gui --disable-tests'                                                |
 | ln             | node  | string  |           | Attach a lightning network node of this implementation (currently only supports 'lnd')                                                                              |
-| ln-image       | node  | string  |           | Specify a lightning network node image from Dockerhub with the format repository/image:tag                                                                          |
-| ln-cb-image    | node  | string  |           | Specify a lnd Circuit Breaker image from Dockerhub with the format repository/image:tag                                                                             |
-| channel        | edge  | number  |           | Indicate that this edge is a lightning channel with this specified capacity                                                                                         |
+| ln_image       | node  | string  |           | Specify a lightning network node image from Dockerhub with the format repository/image:tag                                                                          |
+| ln_cb_image    | node  | string  |           | Specify a lnd Circuit Breaker image from Dockerhub with the format repository/image:tag                                                                             |
+| ln_config      | node  | string  |           | A string of arguments for the lightning network node in command-line format, e.g. '--protocol.wumbo-channels --bitcoin.timelockdelta=80'                            |
+| channel_open   | edge  | string  |           | Indicate that this edge is a lightning channel with these arguments passed to lnd openchannel                                                                       |
+| source_policy  | edge  | string  |           | Update the channel originator policy by passing these arguments passed to lnd updatechanpolicy                                                                      |
+| target_policy  | edge  | string  |           | Update the channel partner policy by passing these arguments passed to lnd updatechanpolicy                                                                         |

Diff for: docs/lightning.md

@@ -17,18 +17,24 @@ Example:
 
 ## Adding LN channels to graph
 
-LN channels are represented in the graphml file as edges with an extra data element
-with key `"channel"` and a value in satoshis representing channel capacity. Note
-that this data element is the only difference between LN channels and regular bitcoin
-p2p connections. The graph will be considered a `MultiDiGraph` and contain two different
-kinds of edges.
+LN channels are represented in the graphml file as edges with extra data elements
+that correspond to arguments to the lnd `openchannel` and `updatechanpolicy` RPC
+commands. The keys are:
+
+- `"channel_open"` (arguments added to `openchannel`)
+- `"target_policy"` or `"source_policy"` (arguments added to `updatechanpolicy`)
+
+The key `"channel_open"` is required to open a LN channel in warnet, and to
+identify an edge in the graphml file as a LN channel.
 
 Example:
 
 ```
-    <edge id="0" source="0" target="1">
-        <data key="channel">100000</data>
-    </edge>
+<edge id="5" source="0" target="1">
+    <data key="channel_open">--local_amt=100000</data>
+    <data key="source_policy">--base_fee_msat=100  --fee_rate_ppm=5  --time_lock_delta=18</data>
+    <data key="target_policy">--base_fee_msat=2200 --fee_rate_ppm=13 --time_lock_delta=20</data>
+</edge>
 ```
 
 A complete example graph with LN nodes and channels is included in the test

Diff for: docs/warcli.md

@@ -105,6 +105,18 @@ options:
 | bitcoin_conf | Path   |            |           |
 | random       | Bool   |            | False     |
 
+### `warcli graph import-json`
+Create a cycle graph with nodes imported from lnd `describegraph` JSON file,
+    and additionally include 7 extra random outbounds per node. Include lightning
+    channels and their policies as well.
+    Returns XML file as string with or without --outfile option.
+
+options:
+| name    | type   | required   | default   |
+|---------|--------|------------|-----------|
+| infile  | Path   | yes        |           |
+| outfile | Path   |            |           |
+
 ### `warcli graph validate`
 Validate a \<graph file> against the schema.
 

Diff for: src/backends/compose/compose_backend.py

@@ -16,7 +16,6 @@
 from warnet.status import RunningStatus
 from warnet.tank import Tank
 from warnet.utils import (
-    default_bitcoin_conf_args,
     get_architecture,
     parse_raw_messages,
 )
@@ -340,24 +339,6 @@ def generate_deployment_file(self, warnet):
             dirs_exist_ok=True,
         )
 
-    def config_args(self, tank: Tank):
-        args = self.default_config_args(tank)
-        if tank.bitcoin_config is not None:
-            args = f"{args} -{tank.bitcoin_config.replace(',', ' -')}"
-        return args
-
-    def default_config_args(self, tank):
-        defaults = default_bitcoin_conf_args()
-        defaults += f" -rpcuser={tank.rpc_user}"
-        defaults += f" -rpcpassword={tank.rpc_password}"
-        defaults += f" -rpcport={tank.rpc_port}"
-        defaults += f" -zmqpubrawblock=tcp://0.0.0.0:{tank.zmqblockport}"
-        defaults += f" -zmqpubrawtx=tcp://0.0.0.0:{tank.zmqtxport}"
-        # connect to initial peers as defined in graph file
-        for dst_index in tank.init_peers:
-            defaults += f" -addnode={self.get_container_name(dst_index, ServiceType.BITCOIN)}"
-        return defaults
-
     def add_services(self, tank: Tank, compose):
         services = compose["services"]
         assert tank.index is not None
@@ -385,13 +366,17 @@ def add_services(self, tank: Tank, compose):
             # Pre-built regular release
             image = f"{DOCKER_REGISTRY}:{tank.version}"
             services[container_name]["image"] = image
+
+        peers = [self.get_container_name(dst_index, ServiceType.BITCOIN) for dst_index in tank.init_peers]
+        args = tank.get_bitcoin_conf(peers)
+
         # Add common bitcoind service details
         services[container_name].update(
             {
                 "container_name": container_name,
                 # logging with json-file to support log shipping with promtail into loki
                 "logging": {"driver": "json-file", "options": {"max-size": "10m"}},
-                "environment": {"BITCOIN_ARGS": self.config_args(tank)},
+                "environment": {"BITCOIN_ARGS": args},
                 "networks": {
                     tank.network_name: {
                         "ipv4_address": f"{tank.ipv4}",
@@ -436,28 +421,11 @@ def add_lnd_service(self, tank, compose):
         ln_cb_container_name = self.get_container_name(tank.index, ServiceType.CIRCUITBREAKER)
         bitcoin_container_name = self.get_container_name(tank.index, ServiceType.BITCOIN)
         # These args are appended to the Dockerfile `ENTRYPOINT ["lnd"]`
-        args = [
-            "--noseedbackup",
-            "--norest",
-            "--debuglevel=debug",
-            "--accept-keysend",
-            "--bitcoin.active",
-            "--bitcoin.regtest",
-            "--bitcoin.node=bitcoind",
-            f"--bitcoind.rpcuser={tank.rpc_user}",
-            f"--bitcoind.rpcpass={tank.rpc_password}",
-            f"--bitcoind.rpchost={tank.ipv4}:{tank.rpc_port}",
-            f"--bitcoind.zmqpubrawblock=tcp://{tank.ipv4}:{tank.zmqblockport}",
-            f"--bitcoind.zmqpubrawtx=tcp://{tank.ipv4}:{tank.zmqtxport}",
-            f"--externalip={tank.lnnode.ipv4}",
-            f"--rpclisten=0.0.0.0:{tank.lnnode.rpc_port}",
-            f"--alias={tank.index}",
-            f"--tlsextradomain={ln_container_name}",
-        ]
+        args = tank.lnnode.get_conf(ln_container_name, bitcoin_container_name)
         services[ln_container_name] = {
             "container_name": ln_container_name,
             "image": tank.lnnode.image,
-            "command": " ".join(args),
+            "command": args,
             "networks": {
                 tank.network_name: {
                     "ipv4_address": f"{tank.lnnode.ipv4}",

Diff for: src/backends/kubernetes/kubernetes_backend.py

@@ -17,7 +17,7 @@
 from kubernetes.stream import stream
 from warnet.status import RunningStatus
 from warnet.tank import Tank
-from warnet.utils import default_bitcoin_conf_args, parse_raw_messages
+from warnet.utils import parse_raw_messages
 
 DOCKER_REGISTRY_CORE = "bitcoindevproject/bitcoin"
 LOCAL_REGISTRY = "warnet/bitcoin-core"
@@ -313,18 +313,6 @@ def generate_deployment_file(self, warnet):
         """
         pass
 
-    def default_bitcoind_config_args(self, tank):
-        defaults = default_bitcoin_conf_args()
-        defaults += f" -rpcuser={tank.rpc_user}"
-        defaults += f" -rpcpassword={tank.rpc_password}"
-        defaults += f" -rpcport={tank.rpc_port}"
-        defaults += f" -zmqpubrawblock=tcp://0.0.0.0:{tank.zmqblockport}"
-        defaults += f" -zmqpubrawtx=tcp://0.0.0.0:{tank.zmqtxport}"
-        # connect to initial peers as defined in graph file
-        for dst_index in tank.init_peers:
-            defaults += f" -addnode={self.get_service_name(dst_index)}"
-        return defaults
-
     def create_bitcoind_container(self, tank: Tank) -> client.V1Container:
         self.log.debug(f"Creating bitcoind container for tank {tank.index}")
         container_name = BITCOIN_CONTAINER_NAME
@@ -356,8 +344,8 @@ def create_bitcoind_container(self, tank: Tank) -> client.V1Container:
         else:
             container_image = f"{DOCKER_REGISTRY_CORE}:{tank.version}"
 
-        bitcoind_options = self.default_bitcoind_config_args(tank)
-        bitcoind_options += f" {tank.bitcoin_config}"
+        peers = [self.get_service_name(dst_index) for dst_index in tank.init_peers]
+        bitcoind_options = tank.get_bitcoin_conf(peers)
         container_env = [client.V1EnvVar(name="BITCOIN_ARGS", value=bitcoind_options)]
 
         bitcoind_container = client.V1Container(
@@ -454,28 +442,12 @@ def create_lnd_container(
         # These args are appended to the Dockerfile `ENTRYPOINT ["lnd"]`
         bitcoind_rpc_host = f"{bitcoind_service_name}.{self.namespace}"
         lightning_dns = f"lightning-{tank.index}.{self.namespace}"
-        args = [
-            "--noseedbackup",
-            "--norest",
-            "--debuglevel=debug",
-            "--accept-keysend",
-            "--bitcoin.active",
-            "--bitcoin.regtest",
-            "--bitcoin.node=bitcoind",
-            f"--bitcoind.rpcuser={tank.rpc_user}",
-            f"--bitcoind.rpcpass={tank.rpc_password}",
-            f"--bitcoind.rpchost={bitcoind_rpc_host}:{tank.rpc_port}",
-            f"--bitcoind.zmqpubrawblock={bitcoind_rpc_host}:{tank.zmqblockport}",
-            f"--bitcoind.zmqpubrawtx={bitcoind_rpc_host}:{tank.zmqtxport}",
-            f"--rpclisten=0.0.0.0:{tank.lnnode.rpc_port}",
-            f"--externalhosts={lightning_dns}",
-            f"--alias={tank.index}",
-        ]
+        args = tank.lnnode.get_conf(lightning_dns, bitcoind_rpc_host)
         self.log.debug(f"Creating lightning container for tank {tank.index} using {args=:}")
         lightning_container = client.V1Container(
             name=LN_CONTAINER_NAME,
             image=tank.lnnode.image,
-            args=args,
+            args=args.split(" "),
             env=[
                 client.V1EnvVar(name="LN_IMPL", value=tank.lnnode.impl),
             ],

Diff for: src/cli/graph.py

@@ -1,3 +1,4 @@
+import json
 from io import BytesIO
 from pathlib import Path
 
@@ -28,7 +29,79 @@ def create(number: int, outfile: Path, version: str, bitcoin_conf: Path, random:
     if outfile:
         file_path = Path(outfile)
         nx.write_graphml(graph, file_path, named_key_ids=True)
-        return f"Generated graph written to file: {outfile}"
+    bio = BytesIO()
+    nx.write_graphml(graph, bio, named_key_ids=True)
+    xml_data = bio.getvalue()
+    print(xml_data.decode("utf-8"))
+
+
+@graph.command()
+@click.argument("infile", type=click.Path())
+@click.option("--outfile", type=click.Path())
+def import_json(infile: Path, outfile: Path):
+    """
+    Create a cycle graph with nodes imported from lnd `describegraph` JSON file,
+    and additionally include 7 extra random outbounds per node. Include lightning
+    channels and their policies as well.
+    Returns XML file as string with or without --outfile option.
+    """
+    with open(infile) as f:
+        json_graph = json.loads(f.read())
+
+    # Start with a connected L1 graph with the right amount of tanks
+    graph = create_cycle_graph(len(json_graph["nodes"]), version=DEFAULT_TAG, bitcoin_conf=None, random_version=False)
+
+    # Initialize all the tanks with basic LN node configurations
+    for index, n in enumerate(graph.nodes()):
+        graph.nodes[n]["bitcoin_config"] = f"-uacomment=tank{index:06}"
+        graph.nodes[n]["ln"] = "lnd"
+        graph.nodes[n]["ln_cb_image"] = "pinheadmz/circuitbreaker:278737d"
+        graph.nodes[n]["ln_config"] = "--protocol.wumbo-channels"
+
+    # Save a map of LN pubkey -> Tank index
+    ln_ids = {}
+    for index, node in enumerate(json_graph["nodes"]):
+        ln_ids[node["id"]] = index
+
+    # Offset for edge IDs
+    # Note create_cycle_graph() creates L1 edges all with the same id "0"
+    L1_edges = len(graph.edges)
+
+    # Insert LN channels
+    # Ensure channels are in order by channel ID like lnd describegraph output
+    sorted_edges = sorted(json_graph["edges"], key=lambda chan: int(chan['channel_id']))
+    for ln_index, channel in enumerate(sorted_edges):
+        src = ln_ids[channel["node1_pub"]]
+        tgt = ln_ids[channel["node2_pub"]]
+        cap = int(channel["capacity"])
+        push = cap // 2
+        openp = f"--local_amt={cap} --push_amt={push}"
+        srcp = ""
+        tgtp = ""
+        if channel["node1_policy"]:
+            srcp += f" --base_fee_msat={channel['node1_policy']['fee_base_msat']}"
+            srcp += f" --fee_rate_ppm={channel['node1_policy']['fee_rate_milli_msat']}"
+            srcp += f" --time_lock_delta={channel['node1_policy']['time_lock_delta']}"
+            srcp += f" --min_htlc_msat={channel['node1_policy']['min_htlc']}"
+            srcp += f" --max_htlc_msat={push * 1000}"
+        if channel["node2_policy"]:
+            tgtp += f" --base_fee_msat={channel['node2_policy']['fee_base_msat']}"
+            tgtp += f" --fee_rate_ppm={channel['node2_policy']['fee_rate_milli_msat']}"
+            tgtp += f" --time_lock_delta={channel['node2_policy']['time_lock_delta']}"
+            tgtp += f" --min_htlc_msat={channel['node2_policy']['min_htlc']}"
+            tgtp += f" --max_htlc_msat={push * 1000}"
+
+        graph.add_edge(
+            src,
+            tgt,
+            key = ln_index+L1_edges,
+            channel_open = openp,
+            source_policy = srcp,
+            target_policy = tgtp)
+
+    if outfile:
+        file_path = Path(outfile)
+        nx.write_graphml(graph, file_path, named_key_ids=True)
     bio = BytesIO()
     nx.write_graphml(graph, bio, named_key_ids=True)
     xml_data = bio.getvalue()
@@ -42,5 +115,5 @@ def validate(graph: Path):
     Validate a <graph file> against the schema.
     """
     with open(graph) as f:
-        graph = nx.parse_graphml(f.read(), node_type=int)
+        graph = nx.parse_graphml(f.read(), node_type=int, force_multigraph=True)
     return validate_graph_schema(graph)