From a3ca3fb047fd79088dbaef5765786f28bab9c875 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 14:59:57 +1030 Subject: [PATCH 01/36] common/bolt11: fix 32-bit compilation. Fixes d9fed06b900368e59f4d1f432b87d40fd28ce8d3: ``` common/bolt11.c:868:31: error: format specifies type 'size_t' (aka 'unsigned long') but the argument has type 'u64' (aka 'unsigned long long') [-Werror,-Wformat] bech32_charset[type], field_len); ^~~~~~~~~ ``` Signed-off-by: Rusty Russell --- common/bolt11.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/common/bolt11.c b/common/bolt11.c index 4b00e8535002..c2c7c3db136e 100644 --- a/common/bolt11.c +++ b/common/bolt11.c @@ -826,7 +826,8 @@ struct bolt11 *bolt11_decode_nosig(const tal_t *ctx, const char *str, while (data_len > 520 / 5) { const char *problem = NULL; - u64 type, field_len; + u64 type, field_len64; + size_t field_len; const struct decoder *decoder; /* BOLT #11: @@ -841,15 +842,21 @@ struct bolt11 *bolt11_decode_nosig(const tal_t *ctx, const char *str, if (err) return decode_fail(b11, fail, "Can't get tag: %s", err); - err = pull_uint(&hu5, &data, &data_len, &field_len, 10); + err = pull_uint(&hu5, &data, &data_len, &field_len64, 10); if (err) return decode_fail(b11, fail, "Can't get length: %s", err); /* Can't exceed total data remaining. */ - if (field_len > data_len) + if (field_len64 > data_len) return decode_fail(b11, fail, "%c: truncated", bech32_charset[type]); + + /* These are different types on 32 bit! But since data_len is + * also size_t, above check ensures this will fit. */ + field_len = field_len64; + assert(field_len == field_len64); + /* Do this now: the decode function fixes up the data ptr */ data_len -= field_len; From f2f05117aa7ecfe2a255d06a2c543a9c89010f36 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 16 Nov 2022 13:38:51 -0600 Subject: [PATCH 02/36] pytest: gossipd: test resumption of pruned channels --- tests/test_gossip.py | 69 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/test_gossip.py b/tests/test_gossip.py index 5793192912fd..053749e1b9a5 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -2223,3 +2223,72 @@ def test_gossip_private_updates(node_factory, bitcoind): l1.restart() wait_for(lambda: l1.daemon.is_in_log(r'gossip_store_compact_offline: 5 deleted, 3 copied')) + + +@pytest.mark.developer("Needs --dev-fast-gossip, --dev-fast-gossip-prune") +@pytest.mark.xfail(strict=True) +def test_channel_resurrection(node_factory, bitcoind): + """When a node goes offline long enough to prune a channel, the + channel_announcement should be retained in case the node comes back online. + """ + opts = {'dev-fast-gossip-prune': None, + 'may_reconnect': True} + l1, l2 = node_factory.get_nodes(2, opts=opts) + opts.update({'log-level': 'debug'}) + l3, = node_factory.get_nodes(1, opts=opts) + l1.rpc.connect(l2.info['id'], 'localhost', l2.port) + l3.rpc.connect(l2.info['id'], 'localhost', l2.port) + scid, _ = l1.fundchannel(l2, 10**6, True, True) + bitcoind.generate_block(6) + sync_blockheight(bitcoind, [l1, l2, l3]) + l3.wait_channel_active(scid) + start_time = int(time.time()) + # Channel_update should now be refreshed. + refresh_due = start_time + 44 + prune_due = start_time + 61 + l2.rpc.call('dev-gossip-set-time', [refresh_due]) + l3.rpc.call('dev-gossip-set-time', [refresh_due]) + # Automatic reconnect is too fast, so shutdown l1 instead of disconnecting + l1.stop() + l2.daemon.wait_for_log('Sending keepalive channel_update') + l3.daemon.wait_for_log('Received channel_update for channel 103x1') + # Wait for the next pruning cycle + l2.rpc.call('dev-gossip-set-time', [prune_due]) + l3.rpc.call('dev-gossip-set-time', [prune_due]) + # Make sure l1 is recognized as disconnected + wait_for(lambda: only_one(l2.rpc.listpeers(l1.info['id'])['peers'])['connected'] is False) + # Wait for the channel to be pruned. + l3.daemon.wait_for_log("Pruning channel") + assert l3.rpc.listchannels()['channels'] == [] + l1.start() + time.sleep(1) + l1.rpc.call('dev-gossip-set-time', [prune_due]) + time.sleep(1) + l1.rpc.call('dev-gossip-set-time', [prune_due]) + wait_for(lambda: [c['active'] for c in l2.rpc.listchannels()['channels']] == [True, True]) + l1.rpc.call('dev-gossip-set-time', [prune_due + 30]) + l2.rpc.call('dev-gossip-set-time', [prune_due + 30]) + l3.rpc.call('dev-gossip-set-time', [prune_due + 30]) + # l2 should recognize its own channel as announceable + wait_for(lambda: [[c['public'], c['active']] for c in l2.rpc.listchannels()['channels']] == [[True, True], [True, True]], timeout=30) + # l3 should be able to recover the zombie channel + wait_for(lambda: [c['active'] for c in l3.rpc.listchannels()['channels']] == [True, True], timeout=30) + + # Now test spending the outpoint and removing a zombie channel from the store. + l2.stop() + prune_again = prune_due + 91 + l1.rpc.call('dev-gossip-set-time', [prune_again]) + l3.rpc.call('dev-gossip-set-time', [prune_again]) + l3.daemon.wait_for_log("Pruning channel") + txid = l1.rpc.close(l2.info['id'], 1)['txid'] + bitcoind.generate_block(13, txid) + l3.daemon.wait_for_log(f"Deleting channel {scid} due to the funding " + "outpoint being spent", 30) + # gossip_store is cleaned of zombie channels once outpoint is spent. + gs_path = os.path.join(l3.daemon.lightning_dir, TEST_NETWORK, 'gossip_store') + gs = subprocess.run(['devtools/dump-gossipstore', '--print-deleted', gs_path], + check=True, timeout=TIMEOUT, stdout=subprocess.PIPE) + print(gs.stdout.decode()) + for l in gs.stdout.decode().splitlines(): + if "ZOMBIE" in l: + assert ("DELETED" in l) From ed4815527aba7a3d11bd9f33b441372edd56310e Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 16 Nov 2022 16:36:25 -0600 Subject: [PATCH 03/36] gossipd: avoid gossipd crash due to double freeing timer --- gossipd/gossip_generation.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gossipd/gossip_generation.c b/gossipd/gossip_generation.c index 755a887d2c37..f95405c69034 100644 --- a/gossipd/gossip_generation.c +++ b/gossipd/gossip_generation.c @@ -342,7 +342,8 @@ static void update_own_node_announcement_after_startup(struct daemon *daemon) static void force_self_nannounce_regen(struct daemon *daemon) { struct node *self = get_node(daemon->rstate, &daemon->id); - + /* Clear timer pointer now. */ + daemon->node_announce_regen_timer = NULL; /* No channels left? We'll restart timer once we have one. */ if (!self || !self->bcast.index) return; From 6bff10cd40a886594f16f3fe906ccc6bef7f567f Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Fri, 16 Dec 2022 10:25:47 -0600 Subject: [PATCH 04/36] gossip_store: add a flag for zombie entries This will allow gossipd to store and persist gossip for channels rather than deleting them entirely when the channels are pruned from the network. --- common/gossip_store.h | 5 +++++ devtools/dump-gossipstore.c | 10 +++++---- gossipd/gossip_store.c | 22 ++++++++++--------- gossipd/gossip_store.h | 4 +++- gossipd/routing.c | 7 +++--- gossipd/test/run-check_channel_announcement.c | 3 ++- gossipd/test/run-txout_failure.c | 3 ++- tests/test_gossip.py | 10 ++++----- 8 files changed, 39 insertions(+), 25 deletions(-) diff --git a/common/gossip_store.h b/common/gossip_store.h index e74f7cba4718..4ddf4289eda9 100644 --- a/common/gossip_store.h +++ b/common/gossip_store.h @@ -39,6 +39,11 @@ struct gossip_rcvd_filter; */ #define GOSSIP_STORE_LEN_RATELIMIT_BIT 0x20000000U +/** + * Bit used to mark a channel announcement as inactive (needs channel updates.) + */ +#define GOSSIP_STORE_LEN_ZOMBIE_BIT 0x10000000U + /** * Full flags mask */ diff --git a/devtools/dump-gossipstore.c b/devtools/dump-gossipstore.c index 6d4d8523c40f..80a9fef823d0 100644 --- a/devtools/dump-gossipstore.c +++ b/devtools/dump-gossipstore.c @@ -12,7 +12,7 @@ /* Current versions we support */ #define GSTORE_MAJOR 0 -#define GSTORE_MINOR 10 +#define GSTORE_MINOR 12 int main(int argc, char *argv[]) { @@ -67,12 +67,13 @@ int main(int argc, char *argv[]) struct short_channel_id scid; u32 msglen = be32_to_cpu(hdr.len); u8 *msg, *inner; - bool deleted, push, ratelimit; + bool deleted, push, ratelimit, zombie; u32 blockheight; deleted = (msglen & GOSSIP_STORE_LEN_DELETED_BIT); push = (msglen & GOSSIP_STORE_LEN_PUSH_BIT); ratelimit = (msglen & GOSSIP_STORE_LEN_RATELIMIT_BIT); + zombie = (msglen & GOSSIP_STORE_LEN_ZOMBIE_BIT); msglen &= GOSSIP_STORE_LEN_MASK; msg = tal_arr(NULL, u8, msglen); @@ -83,10 +84,11 @@ int main(int argc, char *argv[]) != crc32c(be32_to_cpu(hdr.timestamp), msg, msglen)) warnx("Checksum verification failed"); - printf("%zu: %s%s%s", off, + printf("%zu: %s%s%s%s", off, deleted ? "DELETED " : "", push ? "PUSH " : "", - ratelimit ? "RATE-LIMITED " : ""); + ratelimit ? "RATE-LIMITED " : "", + zombie ? "ZOMBIE " : ""); if (print_timestamp) printf("T=%u ", be32_to_cpu(hdr.timestamp)); if (deleted && !print_deleted) { diff --git a/gossipd/gossip_store.c b/gossipd/gossip_store.c index b1e2ffebc34b..b60e77d5a4cd 100644 --- a/gossipd/gossip_store.c +++ b/gossipd/gossip_store.c @@ -17,8 +17,8 @@ #include #define GOSSIP_STORE_TEMP_FILENAME "gossip_store.tmp" -/* We write it as major version 0, minor version 11 */ -#define GOSSIP_STORE_VER ((0 << 5) | 11) +/* We write it as major version 0, minor version 12 */ +#define GOSSIP_STORE_VER ((0 << 5) | 12) struct gossip_store { /* This is false when we're loading */ @@ -73,7 +73,7 @@ static ssize_t gossip_pwritev(int fd, const struct iovec *iov, int iovcnt, #endif /* !HAVE_PWRITEV */ static bool append_msg(int fd, const u8 *msg, u32 timestamp, - bool push, bool spam, u64 *len) + bool push, bool zombie, bool spam, u64 *len) { struct gossip_hdr hdr; u32 msglen; @@ -88,6 +88,8 @@ static bool append_msg(int fd, const u8 *msg, u32 timestamp, hdr.len |= CPU_TO_BE32(GOSSIP_STORE_LEN_PUSH_BIT); if (spam) hdr.len |= CPU_TO_BE32(GOSSIP_STORE_LEN_RATELIMIT_BIT); + if (zombie) + hdr.len |= CPU_TO_BE32(GOSSIP_STORE_LEN_ZOMBIE_BIT); hdr.crc = cpu_to_be32(crc32c(timestamp, msg, msglen)); hdr.timestamp = cpu_to_be32(timestamp); @@ -248,7 +250,7 @@ static u32 gossip_store_compact_offline(struct routing_state *rstate) oldlen = lseek(old_fd, SEEK_END, 0); newlen = lseek(new_fd, SEEK_END, 0); append_msg(old_fd, towire_gossip_store_ended(tmpctx, newlen), - 0, true, false, &oldlen); + 0, true, false, false, &oldlen); close(old_fd); status_debug("gossip_store_compact_offline: %zu deleted, %zu copied", deleted, count); @@ -530,7 +532,7 @@ bool gossip_store_compact(struct gossip_store *gs) /* Write end marker now new one is ready */ append_msg(gs->fd, towire_gossip_store_ended(tmpctx, len), - 0, true, false, &gs->len); + 0, true, false, false, &gs->len); gs->count = count; gs->deleted = 0; @@ -550,7 +552,7 @@ bool gossip_store_compact(struct gossip_store *gs) } u64 gossip_store_add(struct gossip_store *gs, const u8 *gossip_msg, - u32 timestamp, bool push, + u32 timestamp, bool push, bool zombie, bool spam, const u8 *addendum) { u64 off = gs->len; @@ -558,12 +560,12 @@ u64 gossip_store_add(struct gossip_store *gs, const u8 *gossip_msg, /* Should never get here during loading! */ assert(gs->writable); - if (!append_msg(gs->fd, gossip_msg, timestamp, push, spam, &gs->len)) { + if (!append_msg(gs->fd, gossip_msg, timestamp, push, zombie, spam, &gs->len)) { status_broken("Failed writing to gossip store: %s", strerror(errno)); return 0; } - if (addendum && !append_msg(gs->fd, addendum, 0, false, false, &gs->len)) { + if (addendum && !append_msg(gs->fd, addendum, 0, false, false, false, &gs->len)) { status_broken("Failed writing addendum to gossip store: %s", strerror(errno)); return 0; @@ -580,7 +582,7 @@ u64 gossip_store_add_private_update(struct gossip_store *gs, const u8 *update) /* A local update for an unannounced channel: not broadcastable, but * otherwise the same as a normal channel_update */ const u8 *pupdate = towire_gossip_store_private_update(tmpctx, update); - return gossip_store_add(gs, pupdate, 0, false, false, NULL); + return gossip_store_add(gs, pupdate, 0, false, false, false, NULL); } /* Returns index of following entry. */ @@ -640,7 +642,7 @@ void gossip_store_mark_channel_deleted(struct gossip_store *gs, const struct short_channel_id *scid) { gossip_store_add(gs, towire_gossip_store_delete_chan(tmpctx, scid), - 0, false, false, NULL); + 0, false, false, false, NULL); } const u8 *gossip_store_get(const tal_t *ctx, diff --git a/gossipd/gossip_store.h b/gossipd/gossip_store.h index 44a0b4d303bc..67ed9dc798cc 100644 --- a/gossipd/gossip_store.h +++ b/gossipd/gossip_store.h @@ -41,11 +41,13 @@ u64 gossip_store_add_private_update(struct gossip_store *gs, const u8 *update); * @timestamp: the timestamp for filtering of this messsage. * @push: true if this should be sent to peers despite any timestamp filters. * @spam: true if this message is rate-limited and squelched to peers. + * @zombie: true if this channel is missing a current channel_update. * @addendum: another message to append immediately after this * (for appending amounts to channel_announcements for internal use). */ u64 gossip_store_add(struct gossip_store *gs, const u8 *gossip_msg, - u32 timestamp, bool push, bool spam, const u8 *addendum); + u32 timestamp, bool push, bool zombie, bool spam, + const u8 *addendum); /** diff --git a/gossipd/routing.c b/gossipd/routing.c index 3df90bdc602b..d8364fef10f4 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -1797,7 +1797,7 @@ bool routing_add_node_announcement(struct routing_state *rstate, = gossip_store_add(rstate->gs, msg, timestamp, node_id_eq(&node_id, &rstate->local_id), - spam, NULL); + false, spam, NULL); if (node->bcast.timestamp > rstate->last_timestamp && node->bcast.timestamp < time_now().ts.tv_sec) rstate->last_timestamp = node->bcast.timestamp; @@ -2025,7 +2025,8 @@ bool routing_add_private_channel(struct routing_state *rstate, u8 *msg = towire_gossip_store_private_channel(tmpctx, capacity, chan_ann); - index = gossip_store_add(rstate->gs, msg, 0, false, false, NULL); + index = gossip_store_add(rstate->gs, msg, 0, false, false, + false, NULL); } chan->bcast.index = index; return true; @@ -2170,7 +2171,7 @@ void routing_channel_spent(struct routing_state *rstate, /* Save to gossip_store in case we restart */ msg = towire_gossip_store_chan_dying(tmpctx, &chan->scid, deadline); - index = gossip_store_add(rstate->gs, msg, 0, false, false, NULL); + index = gossip_store_add(rstate->gs, msg, 0, false, false, false, NULL); /* Remember locally so we can kill it in 12 blocks */ status_debug("channel %s closing soon due" diff --git a/gossipd/test/run-check_channel_announcement.c b/gossipd/test/run-check_channel_announcement.c index 0e983618e024..bd3cf97dcf2a 100644 --- a/gossipd/test/run-check_channel_announcement.c +++ b/gossipd/test/run-check_channel_announcement.c @@ -61,7 +61,8 @@ bool cupdate_different(struct gossip_store *gs UNNEEDED, { fprintf(stderr, "cupdate_different called!\n"); abort(); } /* Generated stub for gossip_store_add */ u64 gossip_store_add(struct gossip_store *gs UNNEEDED, const u8 *gossip_msg UNNEEDED, - u32 timestamp UNNEEDED, bool push UNNEEDED, bool spam UNNEEDED, const u8 *addendum UNNEEDED) + u32 timestamp UNNEEDED, bool push UNNEEDED, bool zombie UNNEEDED, bool spam UNNEEDED, + const u8 *addendum UNNEEDED) { fprintf(stderr, "gossip_store_add called!\n"); abort(); } /* Generated stub for gossip_store_add_private_update */ u64 gossip_store_add_private_update(struct gossip_store *gs UNNEEDED, const u8 *update UNNEEDED) diff --git a/gossipd/test/run-txout_failure.c b/gossipd/test/run-txout_failure.c index 598f11ff9d27..5db4ef8f3c0f 100644 --- a/gossipd/test/run-txout_failure.c +++ b/gossipd/test/run-txout_failure.c @@ -32,7 +32,8 @@ bool cupdate_different(struct gossip_store *gs UNNEEDED, { fprintf(stderr, "cupdate_different called!\n"); abort(); } /* Generated stub for gossip_store_add */ u64 gossip_store_add(struct gossip_store *gs UNNEEDED, const u8 *gossip_msg UNNEEDED, - u32 timestamp UNNEEDED, bool push UNNEEDED, bool spam UNNEEDED, const u8 *addendum UNNEEDED) + u32 timestamp UNNEEDED, bool push UNNEEDED, bool zombie UNNEEDED, bool spam UNNEEDED, + const u8 *addendum UNNEEDED) { fprintf(stderr, "gossip_store_add called!\n"); abort(); } /* Generated stub for gossip_store_add_private_update */ u64 gossip_store_add_private_update(struct gossip_store *gs UNNEEDED, const u8 *update UNNEEDED) diff --git a/tests/test_gossip.py b/tests/test_gossip.py index 053749e1b9a5..b15cda03dd80 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -1159,7 +1159,7 @@ def test_gossip_store_load(node_factory): """Make sure we can read canned gossip store""" l1 = node_factory.get_node(start=False) with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("0c" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1217,7 +1217,7 @@ def test_gossip_store_load_announce_before_update(node_factory): """Make sure we can read canned gossip store with node_announce before update. This happens when a channel_update gets replaced, leaving node_announce before it""" l1 = node_factory.get_node(start=False) with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("0c" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1262,7 +1262,7 @@ def test_gossip_store_load_amount_truncated(node_factory): """Make sure we can read canned gossip store with truncated amount""" l1 = node_factory.get_node(start=False, allow_broken_log=True) with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("0c" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1727,7 +1727,7 @@ def test_gossip_store_load_no_channel_update(node_factory): # A channel announcement with no channel_update. with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("0b" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("0c" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1754,7 +1754,7 @@ def test_gossip_store_load_no_channel_update(node_factory): l1.rpc.call('dev-compact-gossip-store') with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), "rb") as f: - assert bytearray(f.read()) == bytearray.fromhex("0b") + assert bytearray(f.read()) == bytearray.fromhex("0c") @pytest.mark.developer("gossip without DEVELOPER=1 is slow") From 1bae8cd28a30ff7762789901ed3f0c245cd99892 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Fri, 16 Dec 2022 12:38:23 -0600 Subject: [PATCH 05/36] gossipd: zombify inactive channels instead of pruning Though BOLT 7 says a channel may be pruned when one side becomes inactive and fails to refresh their channel_update, in practice, the channel_announcement can be difficult to recover if deleted entirely. Here the channel_announcement is tagged as zombie such that gossip_store consumers may safely ignore it, but it may be retained should the channel come back online in the future. Node_announcements and channel_updates may also be retained in such a fashion until the channel is ready to be resurrected. Changelog-Fixed: Pruned channels are more reliably restored. --- common/gossmap.c | 3 + gossipd/gossip_store.c | 90 +++++++++ gossipd/gossip_store.h | 14 ++ gossipd/gossipd.c | 13 ++ gossipd/routing.c | 185 +++++++++++++++++- gossipd/routing.h | 3 + gossipd/test/run-check_channel_announcement.c | 12 ++ gossipd/test/run-txout_failure.c | 12 ++ 8 files changed, 322 insertions(+), 10 deletions(-) diff --git a/common/gossmap.c b/common/gossmap.c index af541169a4b6..31176ccf8d49 100644 --- a/common/gossmap.c +++ b/common/gossmap.c @@ -620,6 +620,9 @@ static bool map_catchup(struct gossmap *map, size_t *num_rejected) if (be32_to_cpu(ghdr.len) & GOSSIP_STORE_LEN_DELETED_BIT) continue; + if (be32_to_cpu(ghdr.len) & GOSSIP_STORE_LEN_ZOMBIE_BIT) + continue; + /* Partial write, this can happen. */ if (map->map_end + reclen > map->map_size) break; diff --git a/gossipd/gossip_store.c b/gossipd/gossip_store.c index b60e77d5a4cd..a8a3b81a5483 100644 --- a/gossipd/gossip_store.c +++ b/gossipd/gossip_store.c @@ -645,6 +645,96 @@ void gossip_store_mark_channel_deleted(struct gossip_store *gs, 0, false, false, false, NULL); } +/* Marks the length field of a channel_announcement with the zombie flag bit */ +void gossip_store_mark_channel_zombie(struct gossip_store *gs, + struct broadcastable *bcast) +{ + beint32_t belen; + u32 index = bcast->index; + + /* Should never get here during loading! */ + assert(gs->writable); + + assert(index); + +#if DEVELOPER + const u8 *msg = gossip_store_get(tmpctx, gs, index); + assert(fromwire_peektype(msg) == WIRE_CHANNEL_ANNOUNCEMENT); +#endif + + if (pread(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "Failed reading len to zombie channel @%u: %s", + index, strerror(errno)); + + assert((be32_to_cpu(belen) & GOSSIP_STORE_LEN_DELETED_BIT) == 0); + belen |= cpu_to_be32(GOSSIP_STORE_LEN_ZOMBIE_BIT); + if (pwrite(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "Failed writing len to zombie channel @%u: %s", + index, strerror(errno)); +} + +/* Marks the length field of a channel_update with the zombie flag bit */ +void gossip_store_mark_cupdate_zombie(struct gossip_store *gs, + struct broadcastable *bcast) +{ + beint32_t belen; + u32 index = bcast->index; + + /* Should never get here during loading! */ + assert(gs->writable); + + assert(index); + +#if DEVELOPER + const u8 *msg = gossip_store_get(tmpctx, gs, index); + assert(fromwire_peektype(msg) == WIRE_CHANNEL_UPDATE); +#endif + + if (pread(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "Failed reading len to zombie channel update @%u: %s", + index, strerror(errno)); + + assert((be32_to_cpu(belen) & GOSSIP_STORE_LEN_DELETED_BIT) == 0); + belen |= cpu_to_be32(GOSSIP_STORE_LEN_ZOMBIE_BIT); + if (pwrite(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "Failed writing len to zombie channel update @%u: %s", + index, strerror(errno)); +} + +/* Marks the length field of a node_announcement with the zombie flag bit */ +void gossip_store_mark_nannounce_zombie(struct gossip_store *gs, + struct broadcastable *bcast) +{ + beint32_t belen; + u32 index = bcast->index; + + /* Should never get here during loading! */ + assert(gs->writable); + + assert(index); + +#if DEVELOPER + const u8 *msg = gossip_store_get(tmpctx, gs, index); + assert(fromwire_peektype(msg) == WIRE_NODE_ANNOUNCEMENT); +#endif + + if (pread(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "Failed reading len to zombie node announcement @%u: %s", + index, strerror(errno)); + + assert((be32_to_cpu(belen) & GOSSIP_STORE_LEN_DELETED_BIT) == 0); + belen |= cpu_to_be32(GOSSIP_STORE_LEN_ZOMBIE_BIT); + if (pwrite(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "Failed writing len to zombie channel update @%u: %s", + index, strerror(errno)); +} + const u8 *gossip_store_get(const tal_t *ctx, struct gossip_store *gs, u64 offset) diff --git a/gossipd/gossip_store.h b/gossipd/gossip_store.h index 67ed9dc798cc..e3847e9bcda0 100644 --- a/gossipd/gossip_store.h +++ b/gossipd/gossip_store.h @@ -66,6 +66,20 @@ void gossip_store_delete(struct gossip_store *gs, void gossip_store_mark_channel_deleted(struct gossip_store *gs, const struct short_channel_id *scid); +/* + * Marks the length field of a channel announcement with a zombie flag bit. + * This allows the channel_announcement to be retained in the store while + * waiting for channel updates to reactivate it. + */ +void gossip_store_mark_channel_zombie(struct gossip_store *gs, + struct broadcastable *bcast); + +void gossip_store_mark_cupdate_zombie(struct gossip_store *gs, + struct broadcastable *bcast); + +void gossip_store_mark_nannounce_zombie(struct gossip_store *gs, + struct broadcastable *bcast); + /** * Direct store accessor: loads gossip msg back from store. * diff --git a/gossipd/gossipd.c b/gossipd/gossipd.c index e6ec91a7422f..c2e35e05b409 100644 --- a/gossipd/gossipd.c +++ b/gossipd/gossipd.c @@ -322,17 +322,30 @@ static void handle_local_private_channel(struct daemon *daemon, const u8 *msg) u8 *features; struct short_channel_id scid; const u8 *cannounce; + struct chan *zombie; if (!fromwire_gossipd_local_private_channel(msg, msg, &id, &capacity, &scid, &features)) master_badmsg(WIRE_GOSSIPD_LOCAL_PRIVATE_CHANNEL, msg); + status_debug("received private channel announcement from channeld for %s", + type_to_string(tmpctx, struct short_channel_id, &scid)); cannounce = private_channel_announcement(tmpctx, &scid, &daemon->id, &id, features); + /* If there is already a zombie announcement for this channel in the + * store we can disregard this one. */ + zombie = get_channel(daemon->rstate, &scid); + if (zombie && (zombie->half[0].zombie || zombie->half[1].zombie)){ + status_debug("received channel announcement for %s," + " but it is a zombie; discarding", + type_to_string(tmpctx, struct short_channel_id, + &scid)); + return; + } if (!routing_add_private_channel(daemon->rstate, &id, capacity, cannounce, 0)) { diff --git a/gossipd/routing.c b/gossipd/routing.c index d8364fef10f4..3f2d523368a5 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -394,6 +395,25 @@ static bool node_has_public_channels(struct node *node) return false; } +static bool is_chan_zombie(struct chan *chan) +{ + if (chan->half[0].zombie || chan->half[1].zombie) + return true; + return false; +} + +static bool is_node_zombie(struct node* node) +{ + struct chan_map_iter i; + struct chan *c; + + for (c = first_chan(node, &i); c; c = next_chan(node, &i)) { + if (!is_chan_zombie(c)) + return false; + } + return true; +} + /* We can *send* a channel_announce for a channel attached to this node: * we only send once we have a channel_update. */ static bool node_has_broadcastable_channels(struct node *node) @@ -404,8 +424,8 @@ static bool node_has_broadcastable_channels(struct node *node) for (c = first_chan(node, &i); c; c = next_chan(node, &i)) { if (!is_chan_public(c)) continue; - if (is_halfchan_defined(&c->half[0]) - || is_halfchan_defined(&c->half[1])) + if ((is_halfchan_defined(&c->half[0]) + || is_halfchan_defined(&c->half[1])) && !is_chan_zombie(c)) return true; } return false; @@ -444,6 +464,7 @@ static void force_node_announce_rexmit(struct routing_state *rstate, node->bcast.timestamp, is_local, false, + false, NULL); if (node->rgraph.index == initial_bcast_index){ node->rgraph.index = node->bcast.index; @@ -457,6 +478,7 @@ static void force_node_announce_rexmit(struct routing_state *rstate, node->rgraph.timestamp, is_local, false, + false, NULL); } } @@ -553,6 +575,7 @@ static void init_half_chan(struct routing_state *rstate, broadcastable_init(&c->bcast); broadcastable_init(&c->rgraph); c->tokens = TOKEN_MAX; + c->zombie = false; } static void bad_gossip_order(const u8 *msg, @@ -824,6 +847,7 @@ static void add_channel_announce_to_broadcast(struct routing_state *rstate, chan->bcast.timestamp, is_local, false, + false, addendum); rstate->local_channel_announced |= is_local; } @@ -859,8 +883,9 @@ static void delete_chan_messages_from_store(struct routing_state *rstate, static void remove_channel_from_store(struct routing_state *rstate, struct chan *chan) { - /* Put in tombstone marker */ - gossip_store_mark_channel_deleted(rstate->gs, &chan->scid); + /* Put in tombstone marker. Zombie channels will have one already. */ + if (!is_chan_zombie(chan)) + gossip_store_mark_channel_deleted(rstate->gs, &chan->scid); /* Now delete old entries. */ delete_chan_messages_from_store(rstate, chan); @@ -1290,6 +1315,31 @@ static void delete_spam_update(struct routing_state *rstate, hc->rgraph.timestamp = hc->bcast.timestamp; } +static void resurrect_nannouncements(struct routing_state *rstate, + struct chan *chan) +{ + const u8 *zombie_nann = NULL; + for (int i = 0; i < 2; i++) { + struct node *node = chan->nodes[i]; + /* Use the most recent announcement (could be spam.) */ + zombie_nann = gossip_store_get(tmpctx, rstate->gs, + node->rgraph.index); + /* If there was a spam entry, delete them both. */ + if (node->bcast.index != node->rgraph.index) + gossip_store_delete(rstate->gs, &node->bcast, + WIRE_NODE_ANNOUNCEMENT); + gossip_store_delete(rstate->gs, &node->rgraph, + WIRE_NODE_ANNOUNCEMENT); + node->bcast.index = gossip_store_add(rstate->gs, zombie_nann, + node->rgraph.timestamp, + local_direction(rstate, + chan, NULL), + false, false, NULL); + node->bcast.timestamp = node->rgraph.timestamp; + node->rgraph.index = node->bcast.index; + } +} + bool routing_add_channel_update(struct routing_state *rstate, const u8 *update TAKES, u32 index, @@ -1312,6 +1362,7 @@ bool routing_add_channel_update(struct routing_state *rstate, u8 direction; struct amount_sat sat; bool spam; + bool zombie; /* Make sure we own msg, even if we don't save it. */ if (taken(update)) @@ -1332,6 +1383,7 @@ bool routing_add_channel_update(struct routing_state *rstate, if (chan) { uc = NULL; sat = chan->sat; + zombie = is_chan_zombie(chan); } else { /* Maybe announcement was waiting for this update? */ uc = get_unupdated_channel(rstate, &short_channel_id); @@ -1339,6 +1391,7 @@ bool routing_add_channel_update(struct routing_state *rstate, return false; } sat = uc->sat; + zombie = false; } /* Reject update if the `htlc_maximum_msat` is greater @@ -1468,6 +1521,75 @@ bool routing_add_channel_update(struct routing_state *rstate, return true; } + /* Handle resurrection of zombie channels if the other side of the + * zombie channel has a recent timestamp. */ + if (zombie && timestamp_reasonable(rstate, + chan->half[!direction].bcast.timestamp)) { + status_peer_debug(peer ? &peer->id : NULL, + "Resurrecting zombie channel %s.", + type_to_string(tmpctx, + struct short_channel_id, + &chan->scid)); + const u8 *zombie_announcement = NULL; + const u8 *zombie_addendum = NULL; + const u8 *zombie_update[2] = {NULL, NULL}; + /* Resurrection is a careful process. First delete the zombie- + * flagged channel_announcement which has already been + * tombstoned, and re-add to the store without zombie flag. */ + zombie_announcement = gossip_store_get(tmpctx, rstate->gs, + chan->bcast.index); + u32 offset = tal_count(zombie_announcement) + + sizeof(struct gossip_hdr); + /* The channel_announcement addendum reminds us of its size. */ + zombie_addendum = gossip_store_get(tmpctx, rstate->gs, + chan->bcast.index + offset); + gossip_store_delete(rstate->gs, &chan->bcast, + is_chan_public(chan) + ? WIRE_CHANNEL_ANNOUNCEMENT + : WIRE_GOSSIP_STORE_PRIVATE_CHANNEL); + chan->bcast.index = + gossip_store_add(rstate->gs, zombie_announcement, + chan->bcast.timestamp, + local_direction(rstate, chan, NULL), + false, false, zombie_addendum); + /* Deletion of the old addendum is optional. */ + /* This opposing channel_update has been stashed away. Now that + * there are two valid updates, this one gets restored. */ + /* FIXME: Handle spam case probably needs a helper f'n */ + zombie_update[0] = gossip_store_get(tmpctx, rstate->gs, + chan->half[!direction].bcast.index); + if (chan->half[!direction].bcast.index != chan->half[!direction].rgraph.index) + /* Don't forget the spam channel_update */ + zombie_update[1] = gossip_store_get(tmpctx, rstate->gs, + chan->half[!direction].rgraph.index); + gossip_store_delete(rstate->gs, &chan->half[!direction].bcast, + is_chan_public(chan) + ? WIRE_CHANNEL_UPDATE + : WIRE_GOSSIP_STORE_PRIVATE_UPDATE); + chan->half[!direction].bcast.index = + gossip_store_add(rstate->gs, zombie_update[0], + chan->half[!direction].bcast.timestamp, + local_direction(rstate, chan, NULL), + false, false, NULL); + if (zombie_update[1]) + chan->half[!direction].rgraph.index = + gossip_store_add(rstate->gs, zombie_update[1], + chan->half[!direction].rgraph.timestamp, + local_direction(rstate, chan, NULL), + false, true, NULL); + else + chan->half[!direction].rgraph.index = chan->half[!direction].bcast.index; + + /* If we caught any node_announcements for fully zombie nodes + * (no remaining active channels) handle those as well. */ + resurrect_nannouncements(rstate, chan); + + /* It's a miracle! */ + chan->half[0].zombie = false; + chan->half[1].zombie = false; + zombie = false; + } + /* If we're loading from store, this means we don't re-add to store. */ if (index) { if (!spam) @@ -1477,7 +1599,7 @@ bool routing_add_channel_update(struct routing_state *rstate, hc->rgraph.index = gossip_store_add(rstate->gs, update, timestamp, local_direction(rstate, chan, NULL), - spam, NULL); + zombie, spam, NULL); if (hc->bcast.timestamp > rstate->last_timestamp && hc->bcast.timestamp < time_now().ts.tv_sec) rstate->last_timestamp = hc->bcast.timestamp; @@ -1673,7 +1795,8 @@ bool routing_add_node_announcement(struct routing_state *rstate, node = get_node(rstate, &node_id); - if (node == NULL || !node_has_broadcastable_channels(node)) { + if (node == NULL || (!node_has_broadcastable_channels(node) && + !is_node_zombie(node))) { struct pending_node_announce *pna; /* BOLT #7: * @@ -1797,7 +1920,7 @@ bool routing_add_node_announcement(struct routing_state *rstate, = gossip_store_add(rstate->gs, msg, timestamp, node_id_eq(&node_id, &rstate->local_id), - false, spam, NULL); + is_node_zombie(node), spam, NULL); if (node->bcast.timestamp > rstate->last_timestamp && node->bcast.timestamp < time_now().ts.tv_sec) rstate->last_timestamp = node->bcast.timestamp; @@ -1903,6 +2026,43 @@ u8 *handle_node_announcement(struct routing_state *rstate, const u8 *node_ann, return NULL; } +/* Set zombie flags in gossip_store and tombstone the channel for any + * gossip_store consumers. Remove any orphaned node_announcements. */ +static void zombify_channel(struct gossip_store *gs, struct chan *channel) +{ + struct half_chan *half; + assert(!is_chan_zombie(channel)); + gossip_store_mark_channel_zombie(gs, &channel->bcast); + gossip_store_mark_channel_deleted(gs, &channel->scid); + for (int i = 0; i < 2; i++) { + half = &channel->half[i]; + half->zombie = true; + if (half->bcast.index) { + gossip_store_mark_cupdate_zombie(gs, &half->bcast); + /* Channel may also have a spam entry */ + if (half->bcast.index != half->rgraph.index) + gossip_store_mark_cupdate_zombie(gs, &half->rgraph); + } + } + status_debug("Channel %s zombified", + type_to_string(tmpctx, struct short_channel_id, + &channel->scid)); + + /* If one of the nodes has no remaining active channels, the + * node_announcement should also be stashed. */ + for (int i = 0; i < 2; i++) { + struct node *node = channel->nodes[i]; + if (!is_node_zombie(node) || !node->bcast.index) + continue; + if (node->rgraph.index != node->bcast.index) + gossip_store_mark_nannounce_zombie(gs, &node->rgraph); + gossip_store_mark_nannounce_zombie(gs, &node->bcast); + status_debug("Node %s zombified", + type_to_string(tmpctx, struct node_id, + &node->id)); + } +} + void route_prune(struct routing_state *rstate) { u64 now = gossip_time_now(rstate).ts.tv_sec; @@ -1918,6 +2078,9 @@ void route_prune(struct routing_state *rstate) /* Local-only? Don't prune. */ if (!is_chan_public(chan)) continue; + /* These have been pruned already */ + if (is_chan_zombie(chan)) + continue; /* BOLT #7: * - if the `timestamp` of the latest `channel_update` in @@ -1953,10 +2116,12 @@ void route_prune(struct routing_state *rstate) } } - /* Now free all the chans and maybe even nodes. */ + /* Any channels missing an update are now considered zombies. They may + * come back later, in which case the channel_announcement needs to be + * stashed away for later use. If all remaining channels for a node are + * zombies, the node is zombified too. */ for (size_t i = 0; i < tal_count(pruned); i++) { - remove_channel_from_store(rstate, pruned[i]); - free_chan(rstate, pruned[i]); + zombify_channel(rstate->gs, pruned[i]); } } diff --git a/gossipd/routing.h b/gossipd/routing.h index aa97bf8ea693..b8aa671b7bcd 100644 --- a/gossipd/routing.h +++ b/gossipd/routing.h @@ -29,6 +29,9 @@ struct half_chan { /* Token bucket */ u8 tokens; + + /* Disabled channel waiting for a channel_update from both sides. */ + bool zombie; }; struct chan { diff --git a/gossipd/test/run-check_channel_announcement.c b/gossipd/test/run-check_channel_announcement.c index bd3cf97dcf2a..8875693050a4 100644 --- a/gossipd/test/run-check_channel_announcement.c +++ b/gossipd/test/run-check_channel_announcement.c @@ -86,6 +86,18 @@ const u8 *gossip_store_get_private_update(const tal_t *ctx UNNEEDED, void gossip_store_mark_channel_deleted(struct gossip_store *gs UNNEEDED, const struct short_channel_id *scid UNNEEDED) { fprintf(stderr, "gossip_store_mark_channel_deleted called!\n"); abort(); } +/* Generated stub for gossip_store_mark_channel_zombie */ +void gossip_store_mark_channel_zombie(struct gossip_store *gs UNNEEDED, + struct broadcastable *bcast UNNEEDED) +{ fprintf(stderr, "gossip_store_mark_channel_zombie called!\n"); abort(); } +/* Generated stub for gossip_store_mark_cupdate_zombie */ +void gossip_store_mark_cupdate_zombie(struct gossip_store *gs UNNEEDED, + struct broadcastable *bcast UNNEEDED) +{ fprintf(stderr, "gossip_store_mark_cupdate_zombie called!\n"); abort(); } +/* Generated stub for gossip_store_mark_nannounce_zombie */ +void gossip_store_mark_nannounce_zombie(struct gossip_store *gs UNNEEDED, + struct broadcastable *bcast UNNEEDED) +{ fprintf(stderr, "gossip_store_mark_nannounce_zombie called!\n"); abort(); } /* Generated stub for gossip_store_new */ struct gossip_store *gossip_store_new(struct routing_state *rstate UNNEEDED, struct list_head *peers UNNEEDED) diff --git a/gossipd/test/run-txout_failure.c b/gossipd/test/run-txout_failure.c index 5db4ef8f3c0f..049e7e0edc92 100644 --- a/gossipd/test/run-txout_failure.c +++ b/gossipd/test/run-txout_failure.c @@ -57,6 +57,18 @@ const u8 *gossip_store_get_private_update(const tal_t *ctx UNNEEDED, void gossip_store_mark_channel_deleted(struct gossip_store *gs UNNEEDED, const struct short_channel_id *scid UNNEEDED) { fprintf(stderr, "gossip_store_mark_channel_deleted called!\n"); abort(); } +/* Generated stub for gossip_store_mark_channel_zombie */ +void gossip_store_mark_channel_zombie(struct gossip_store *gs UNNEEDED, + struct broadcastable *bcast UNNEEDED) +{ fprintf(stderr, "gossip_store_mark_channel_zombie called!\n"); abort(); } +/* Generated stub for gossip_store_mark_cupdate_zombie */ +void gossip_store_mark_cupdate_zombie(struct gossip_store *gs UNNEEDED, + struct broadcastable *bcast UNNEEDED) +{ fprintf(stderr, "gossip_store_mark_cupdate_zombie called!\n"); abort(); } +/* Generated stub for gossip_store_mark_nannounce_zombie */ +void gossip_store_mark_nannounce_zombie(struct gossip_store *gs UNNEEDED, + struct broadcastable *bcast UNNEEDED) +{ fprintf(stderr, "gossip_store_mark_nannounce_zombie called!\n"); abort(); } /* Generated stub for memleak_add_helper_ */ void memleak_add_helper_(const tal_t *p UNNEEDED, void (*cb)(struct htable *memtable UNNEEDED, const tal_t *)){ } From 98bfd23fff7d2be8096413209c4ff84a4e5739f3 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 17 Jan 2023 13:21:41 -0600 Subject: [PATCH 06/36] pytest: test_channel_resurrection now succeeds --- tests/test_gossip.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_gossip.py b/tests/test_gossip.py index b15cda03dd80..f4c5b52de02c 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -2226,7 +2226,6 @@ def test_gossip_private_updates(node_factory, bitcoind): @pytest.mark.developer("Needs --dev-fast-gossip, --dev-fast-gossip-prune") -@pytest.mark.xfail(strict=True) def test_channel_resurrection(node_factory, bitcoind): """When a node goes offline long enough to prune a channel, the channel_announcement should be retained in case the node comes back online. From fc4bf39d3a9850210bc447c81a1847816d214d08 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:34:03 +1030 Subject: [PATCH 07/36] doc: remove unused offerout schema. We removed the command for v22.11. Also, we removed the `refund_for` offer parameter, so remove its description from the manpage. Signed-off-by: Rusty Russell --- contrib/msggen/msggen/utils/utils.py | 1 - doc/Makefile | 1 - doc/index.rst | 1 - doc/lightning-decode.7.md | 2 +- doc/lightning-disableoffer.7.md | 5 +- doc/lightning-listoffers.7.md | 2 +- doc/lightning-offer.7.md | 6 +- doc/lightning-offerout.7.md | 102 --------------------------- doc/schemas/offerout.schema.json | 54 -------------- 9 files changed, 5 insertions(+), 169 deletions(-) delete mode 100644 doc/lightning-offerout.7.md delete mode 100644 doc/schemas/offerout.schema.json diff --git a/contrib/msggen/msggen/utils/utils.py b/contrib/msggen/msggen/utils/utils.py index 883757b47b5e..4ef75d26cc2b 100644 --- a/contrib/msggen/msggen/utils/utils.py +++ b/contrib/msggen/msggen/utils/utils.py @@ -83,7 +83,6 @@ def load_jsonrpc_service(schema_dir: str): "ListPays", # "multifundchannel", # "multiwithdraw", - # "offerout", # "offer", # "openchannel_abort", # "openchannel_bump", diff --git a/doc/Makefile b/doc/Makefile index 75a8faa4d426..780ff6b3b434 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -66,7 +66,6 @@ MANPAGES := doc/lightning-cli.1 \ doc/lightning-newaddr.7 \ doc/lightning-notifications.7 \ doc/lightning-offer.7 \ - doc/lightning-offerout.7 \ doc/lightning-openchannel_abort.7 \ doc/lightning-openchannel_bump.7 \ doc/lightning-openchannel_init.7 \ diff --git a/doc/index.rst b/doc/index.rst index 73018062400d..3a7907e198c6 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -95,7 +95,6 @@ Core Lightning Documentation lightning-newaddr lightning-notifications lightning-offer - lightning-offerout lightning-openchannel_abort lightning-openchannel_bump lightning-openchannel_init diff --git a/doc/lightning-decode.7.md b/doc/lightning-decode.7.md index 78a5578ddaa8..c4a17f0a17ca 100644 --- a/doc/lightning-decode.7.md +++ b/doc/lightning-decode.7.md @@ -291,7 +291,7 @@ Rusty Russell <> is mainly responsible. SEE ALSO -------- -lightning-pay(7), lightning-offer(7), lightning-offerout(7), lightning-fetchinvoice(7), lightning-sendinvoice(7), lightning-commando-rune(7) +lightning-pay(7), lightning-offer(7), lightning-fetchinvoice(7), lightning-sendinvoice(7), lightning-commando-rune(7) [BOLT #11](https://github.com/lightning/bolts/blob/master/11-payment-encoding.md) diff --git a/doc/lightning-disableoffer.7.md b/doc/lightning-disableoffer.7.md index 8776532fbd87..f9bb46085524 100644 --- a/doc/lightning-disableoffer.7.md +++ b/doc/lightning-disableoffer.7.md @@ -11,8 +11,7 @@ DESCRIPTION ----------- The **disableoffer** RPC command disables an offer, so that no further -invoices will be given out (if made with lightning-offer(7)) or -invoices accepted (if made with lightning-offerout(7)). +invoices will be given out. We currently don't support deletion of offers, so offers are not forgotten entirely (there may be invoices which refer to this offer). @@ -68,7 +67,7 @@ Rusty Russell <> is mainly responsible. SEE ALSO -------- -lightning-offer(7), lightning-offerout(7), lightning-listoffers(7). +lightning-offer(7), lightning-listoffers(7). RESOURCES --------- diff --git a/doc/lightning-listoffers.7.md b/doc/lightning-listoffers.7.md index 1695a86ce140..ad58ae4fa3e4 100644 --- a/doc/lightning-listoffers.7.md +++ b/doc/lightning-listoffers.7.md @@ -74,7 +74,7 @@ Rusty Russell <> is mainly responsible. SEE ALSO -------- -lightning-offer(7), lightning-offerout(7), lightning-listoffers(7). +lightning-offer(7), lightning-listoffers(7). RESOURCES --------- diff --git a/doc/lightning-offer.7.md b/doc/lightning-offer.7.md index edddee664e06..d2b4ed5e0056 100644 --- a/doc/lightning-offer.7.md +++ b/doc/lightning-offer.7.md @@ -84,10 +84,6 @@ periods. This is encoded in the offer. period which exists. eg. "12" means there are 13 periods, from 0 to 12 inclusive. This is encoded in the offer. -*refund\_for* is the payment\_preimage of a previous (paid) invoice. -This implies *send\_invoice* and *single\_use*. This is encoded in the -offer. - *single\_use* (default false) indicates that the offer is only valid once; we may issue multiple invoices, but as soon as one is paid all other invoices will be expired (i.e. only one person can pay this offer). @@ -128,7 +124,7 @@ Rusty Russell <> is mainly responsible. SEE ALSO -------- -lightning-offerout(7), lightning-listoffers(7), lightning-disableoffer(7). +lightning-listoffers(7), lightning-disableoffer(7). RESOURCES --------- diff --git a/doc/lightning-offerout.7.md b/doc/lightning-offerout.7.md deleted file mode 100644 index b74edba049b5..000000000000 --- a/doc/lightning-offerout.7.md +++ /dev/null @@ -1,102 +0,0 @@ -lightning-offerout -- Command for offering payments -================================================= - -SYNOPSIS --------- - -**(WARNING: experimental-offers only)** - - -**offerout** *amount* *description* [*issuer*] [*label*] [*absolute\_expiry*] [*refund\_for*] - -DESCRIPTION ------------ - -The **offerout** RPC command creates an offer, which is a request to -send an invoice for us to pay (technically, this is referred to as a -`send_invoice` offer to distinguish a normal lightningd-offer(7) -offer). It automatically enables the accepting and payment of -corresponding invoice message (we will only pay once, however!). - -Note that it creates two variants of the offer: a signed and an -unsigned one (which is smaller). Wallets should accept both: the -current specification allows either. - -The *amount* parameter can be the string "any", which creates an offer -that can be paid with any amount (e.g. a donation). Otherwise it can -be a positive value in millisatoshi precision; it can be a whole -number, or a whole number ending in *msat* or *sat*, or a number with -three decimal places ending in *sat*, or a number with 1 to 11 decimal -places ending in *btc*. - -The *description* is a short description of purpose of the offer, -e.g. *withdrawl from ATM*. This value is encoded into the resulting offer and is -viewable by anyone you expose this offer to. It must be UTF-8, and -cannot use *\\u* JSON escape codes. - -The *issuer* is another (optional) field exposed in the offer, and -reflects who is issuing this offer (i.e. you) if appropriate. - -The *label* field is an internal-use name for the offer, which can -be any UTF-8 string. - -The *absolute\_expiry* is optionally the time the offer is valid until, -in seconds since the first day of 1970 UTC. If not set, the offer -remains valid (though it can be deactivated by the issuer of course). -This is encoded in the offer. - -*refund\_for* is a previous (paid) invoice of ours. The -payment\_preimage of this is encoded in the offer, and redemption -requires that the invoice we receive contains a valid signature using -that previous `payer_key`. - -RETURN VALUE ------------- - -[comment]: # (GENERATE-FROM-SCHEMA-START) -On success, an object is returned, containing: - -- **offer\_id** (hex): the id of this offer (merkle hash of non-signature fields) (always 64 characters) -- **active** (boolean): whether this will pay a matching incoming invoice (always *true*) -- **single\_use** (boolean): whether this expires as soon as it's paid out (always *true*) -- **bolt12** (string): the bolt12 encoding of the offer -- **used** (boolean): True if an incoming invoice has been paid (always *false*) -- **created** (boolean): false if the offer already existed -- **label** (string, optional): the (optional) user-specified label - -[comment]: # (GENERATE-FROM-SCHEMA-END) - -On failure, an error is returned and no offer is created. If the -lightning process fails before responding, the caller should use -lightning-listoffers(7) to query whether this offer was created or -not. - -The following error codes may occur: -- -1: Catchall nonspecific error. -- 1000: Offer with this offer\_id already exists. - -NOTES ------ - -The specification allows quantity, recurrence and alternate currencies on -offers which contain `send_invoice`, but these are not implemented here. - -We could also allow multi-use offers, but usually you're only offering to -send money once. - -AUTHOR ------- - -Rusty Russell <> is mainly responsible. - -SEE ALSO --------- - -lightning-sendinvoice(7), lightning-offer(7), lightning-listoffers(7), lightning-disableoffer(7). - -RESOURCES ---------- - -Main web site: - -[comment]: # ( SHA256STAMP:e28527bc56d2b54a77b222376c9280a612f7337c908ee0edcfa56d4d0ca2ac6c) diff --git a/doc/schemas/offerout.schema.json b/doc/schemas/offerout.schema.json deleted file mode 100644 index e0874e094357..000000000000 --- a/doc/schemas/offerout.schema.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": false, - "required": [ - "offer_id", - "active", - "single_use", - "bolt12", - "used", - "created" - ], - "properties": { - "offer_id": { - "type": "hex", - "description": "the id of this offer (merkle hash of non-signature fields)", - "maxLength": 64, - "minLength": 64 - }, - "active": { - "type": "boolean", - "enum": [ - true - ], - "description": "whether this will pay a matching incoming invoice" - }, - "single_use": { - "type": "boolean", - "enum": [ - true - ], - "description": "whether this expires as soon as it's paid out" - }, - "bolt12": { - "type": "string", - "description": "the bolt12 encoding of the offer" - }, - "used": { - "type": "boolean", - "enum": [ - false - ], - "description": "True if an incoming invoice has been paid" - }, - "created": { - "type": "boolean", - "description": "false if the offer already existed" - }, - "label": { - "type": "string", - "description": "the (optional) user-specified label" - } - } -} From 7962de7880cde8374dd7d89d44560c9e94ca5b6a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:35:03 +1030 Subject: [PATCH 08/36] wallet: remove unused TX_ANNOTATION type in transaction_annotations table. We only ever use this table for output and input transactions: indeed, my node doesn't have any annotation types 0. Signed-off-by: Rusty Russell --- common/wallet.h | 4 +--- wallet/wallet.c | 13 ++++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/common/wallet.h b/common/wallet.h index aaeb4fe850a8..5b14b3def8a8 100644 --- a/common/wallet.h +++ b/common/wallet.h @@ -21,10 +21,8 @@ enum wallet_tx_type { TX_CHANNEL_CHEAT = 1024, }; -/* What part of a transaction are we annotating? The entire transaction, an - * input or an output. */ +/* What part of a transaction are we annotating? An input or an output. */ enum wallet_tx_annotation_type { - TX_ANNOTATION = 0, OUTPUT_ANNOTATION = 1, INPUT_ANNOTATION = 2, }; diff --git a/wallet/wallet.c b/wallet/wallet.c index 2822732f17bf..1424dfc3bbc1 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -4863,14 +4863,17 @@ struct wallet_transaction *wallet_transactions_get(struct wallet *w, const tal_t struct tx_annotation *ann; /* Select annotation from array to fill in. */ - if (loc == OUTPUT_ANNOTATION) + switch (loc) { + case OUTPUT_ANNOTATION: ann = &cur->output_annotations[idx]; - else if (loc == INPUT_ANNOTATION) + goto got_ann; + case INPUT_ANNOTATION: ann = &cur->input_annotations[idx]; - else - fatal("Transaction annotations are only available for inputs and outputs. Value %d", loc); + goto got_ann; + } + fatal("Transaction annotations are only available for inputs and outputs. Value %d", loc); - /* cppcheck-suppress uninitvar - false positive on fatal() above */ + got_ann: ann->type = db_col_int(stmt, "annotation_type"); if (!db_col_is_null(stmt, "c.scid")) db_col_scid(stmt, "c.scid", &ann->channel); From 0edd183c0009d49f6ab27f5006b3d57e30d75f99 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:36:03 +1030 Subject: [PATCH 09/36] listtransactions: get rid of per-tx type annotations. We didn't actually populate them properly, and the real annotations are on inputs and outputs. Signed-off-by: Rusty Russell Changelog-EXPERIMENTAL: JSON-RPC: `listtransactions` `channel` and `type` field removed at top level. --- cln-grpc/proto/node.proto | 1 - cln-grpc/src/convert.rs | 1 - cln-rpc/src/model.rs | 2 - contrib/pyln-testing/pyln/testing/grpc2py.py | 2 - doc/lightning-listtransactions.7.md | 5 +- doc/lightning-preapproveinvoice.7.md | 2 +- doc/lightning-preapprovekeysend.7.md | 2 +- doc/schemas/listtransactions.schema.json | 24 ------- lightningd/channel.c | 6 +- lightningd/channel.h | 4 +- lightningd/channel_control.c | 6 +- lightningd/closing_control.c | 2 +- lightningd/dual_open_control.c | 4 +- lightningd/onchain_control.c | 6 +- lightningd/peer_control.c | 6 +- lightningd/peer_htlcs.c | 2 +- lightningd/test/run-invoice-select-inchan.c | 8 +-- onchaind/onchaind.c | 47 +----------- onchaind/onchaind_wire.csv | 1 - onchaind/test/run-grind_feerate-bug.c | 2 +- onchaind/test/run-grind_feerate.c | 2 +- wallet/db.c | 2 + wallet/wallet.c | 75 +------------------- wallet/wallet.h | 24 ------- wallet/walletrpc.c | 22 +----- 25 files changed, 23 insertions(+), 235 deletions(-) diff --git a/cln-grpc/proto/node.proto b/cln-grpc/proto/node.proto index dca6d7e71952..b7386cb25ed1 100644 --- a/cln-grpc/proto/node.proto +++ b/cln-grpc/proto/node.proto @@ -744,7 +744,6 @@ message ListtransactionsTransactions { bytes rawtx = 2; uint32 blockheight = 3; uint32 txindex = 4; - optional string channel = 6; uint32 locktime = 7; uint32 version = 8; repeated ListtransactionsTransactionsInputs inputs = 9; diff --git a/cln-grpc/src/convert.rs b/cln-grpc/src/convert.rs index c8096ae28a3d..df5bb8e7d0bd 100644 --- a/cln-grpc/src/convert.rs +++ b/cln-grpc/src/convert.rs @@ -632,7 +632,6 @@ impl From for pb::ListtransactionsTrans rawtx: hex::decode(&c.rawtx).unwrap(), // Rule #2 for type hex blockheight: c.blockheight, // Rule #2 for type u32 txindex: c.txindex, // Rule #2 for type u32 - channel: c.channel.map(|v| v.to_string()), // Rule #2 for type short_channel_id? locktime: c.locktime, // Rule #2 for type u32 version: c.version, // Rule #2 for type u32 inputs: c.inputs.into_iter().map(|i| i.into()).collect(), // Rule #3 for type ListtransactionsTransactionsInputs diff --git a/cln-rpc/src/model.rs b/cln-rpc/src/model.rs index 38601d811786..45ab1e82ea5d 100644 --- a/cln-rpc/src/model.rs +++ b/cln-rpc/src/model.rs @@ -2632,8 +2632,6 @@ pub mod responses { pub rawtx: String, pub blockheight: u32, pub txindex: u32, - #[serde(skip_serializing_if = "Option::is_none")] - pub channel: Option, pub locktime: u32, pub version: u32, pub inputs: Vec, diff --git a/contrib/pyln-testing/pyln/testing/grpc2py.py b/contrib/pyln-testing/pyln/testing/grpc2py.py index 482dc822d50c..5a4bdc1a35f3 100644 --- a/contrib/pyln-testing/pyln/testing/grpc2py.py +++ b/contrib/pyln-testing/pyln/testing/grpc2py.py @@ -511,8 +511,6 @@ def listtransactions_transactions2py(m): "rawtx": hexlify(m.rawtx), # PrimitiveField in generate_composite "blockheight": m.blockheight, # PrimitiveField in generate_composite "txindex": m.txindex, # PrimitiveField in generate_composite - "type": [str(i) for i in m.type], # ArrayField[composite] in generate_composite - "channel": m.channel, # PrimitiveField in generate_composite "locktime": m.locktime, # PrimitiveField in generate_composite "version": m.version, # PrimitiveField in generate_composite "inputs": [listtransactions_transactions_inputs2py(i) for i in m.inputs], # ArrayField[composite] in generate_composite diff --git a/doc/lightning-listtransactions.7.md b/doc/lightning-listtransactions.7.md index 76619a0a601b..a0f8e72bd12e 100644 --- a/doc/lightning-listtransactions.7.md +++ b/doc/lightning-listtransactions.7.md @@ -45,9 +45,6 @@ On success, an object containing **transactions** is returned. It is an array o - **scriptPubKey** (hex): the scriptPubKey - **type** (string, optional): the purpose of this output (*EXPERIMENTAL\_FEATURES* only) (one of "theirs", "deposit", "withdraw", "channel\_funding", "channel\_mutual\_close", "channel\_unilateral\_close", "channel\_sweep", "channel\_htlc\_success", "channel\_htlc\_timeout", "channel\_penalty", "channel\_unilateral\_cheat") - **channel** (short\_channel\_id, optional): the channel this output is associated with (*EXPERIMENTAL\_FEATURES* only) -- **type** (array of strings, optional): - - Reason we care about this transaction (*EXPERIMENTAL\_FEATURES* only) (one of "theirs", "deposit", "withdraw", "channel\_funding", "channel\_mutual\_close", "channel\_unilateral\_close", "channel\_sweep", "channel\_htlc\_success", "channel\_htlc\_timeout", "channel\_penalty", "channel\_unilateral\_cheat") -- **channel** (short\_channel\_id, optional): the channel this transaction is associated with (*EXPERIMENTAL\_FEATURES* only) [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -106,4 +103,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:450383460036860bfeb65fac98582b4c075d9b6c8df326f22ee1aabde7980d74) +[comment]: # ( SHA256STAMP:4820c0c2f399fd5bec1a960bdc731c131a0fb019f7506df3053ae1bc08705845) diff --git a/doc/lightning-preapproveinvoice.7.md b/doc/lightning-preapproveinvoice.7.md index c39d96dcac3e..e4ff52291d7c 100644 --- a/doc/lightning-preapproveinvoice.7.md +++ b/doc/lightning-preapproveinvoice.7.md @@ -48,4 +48,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:735dd61146b04745f1e884037ead662a386fec2c41e2de1a8698d6bb03f63540) +[comment]: # ( SHA256STAMP:ec98523e094209b75eeeb620d8f2a64409dafe6ba21baf3a89ade514b285d202) diff --git a/doc/lightning-preapprovekeysend.7.md b/doc/lightning-preapprovekeysend.7.md index 533041818214..0f0f776f37d0 100644 --- a/doc/lightning-preapprovekeysend.7.md +++ b/doc/lightning-preapprovekeysend.7.md @@ -58,4 +58,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:735dd61146b04745f1e884037ead662a386fec2c41e2de1a8698d6bb03f63540) +[comment]: # ( SHA256STAMP:ec98523e094209b75eeeb620d8f2a64409dafe6ba21baf3a89ade514b285d202) diff --git a/doc/schemas/listtransactions.schema.json b/doc/schemas/listtransactions.schema.json index 3c34ba896372..e0f75bd66368 100644 --- a/doc/schemas/listtransactions.schema.json +++ b/doc/schemas/listtransactions.schema.json @@ -38,30 +38,6 @@ "type": "u32", "description": "the transaction number within the block" }, - "type": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "theirs", - "deposit", - "withdraw", - "channel_funding", - "channel_mutual_close", - "channel_unilateral_close", - "channel_sweep", - "channel_htlc_success", - "channel_htlc_timeout", - "channel_penalty", - "channel_unilateral_cheat" - ], - "description": "Reason we care about this transaction (*EXPERIMENTAL_FEATURES* only)" - } - }, - "channel": { - "type": "short_channel_id", - "description": "the channel this transaction is associated with (*EXPERIMENTAL_FEATURES* only)" - }, "locktime": { "type": "u32", "description": "The nLocktime for this tx" diff --git a/lightningd/channel.c b/lightningd/channel.c index ec697518d597..5687269bd57e 100644 --- a/lightningd/channel.c +++ b/lightningd/channel.c @@ -239,7 +239,6 @@ struct channel *new_unsaved_channel(struct peer *peer, channel->shutdown_scriptpubkey[REMOTE] = NULL; channel->last_was_revoke = false; channel->last_sent_commit = NULL; - channel->last_tx_type = TX_UNKNOWN; channel->feerate_base = feerate_base; channel->feerate_ppm = feerate_ppm; @@ -452,7 +451,6 @@ struct channel *new_channel(struct peer *peer, u64 dbid, channel->last_tx = tal_steal(channel, last_tx); if (channel->last_tx) { channel->last_tx->chainparams = chainparams; - channel->last_tx_type = TX_UNKNOWN; } channel->last_sig = *last_sig; channel->last_htlc_sigs = tal_steal(channel, last_htlc_sigs); @@ -723,14 +721,12 @@ struct channel *find_channel_by_alias(const struct peer *peer, void channel_set_last_tx(struct channel *channel, struct bitcoin_tx *tx, - const struct bitcoin_signature *sig, - enum wallet_tx_type txtypes) + const struct bitcoin_signature *sig) { assert(tx->chainparams); channel->last_sig = *sig; tal_free(channel->last_tx); channel->last_tx = tal_steal(channel, tx); - channel->last_tx_type = txtypes; } void channel_set_state(struct channel *channel, diff --git a/lightningd/channel.h b/lightningd/channel.h index 9a31157c6903..381794cff0ed 100644 --- a/lightningd/channel.h +++ b/lightningd/channel.h @@ -157,7 +157,6 @@ struct channel { /* Last tx they gave us. */ struct bitcoin_tx *last_tx; - enum wallet_tx_type last_tx_type; struct bitcoin_signature last_sig; const struct bitcoin_signature *last_htlc_sigs; @@ -435,8 +434,7 @@ struct channel *find_channel_by_alias(const struct peer *peer, void channel_set_last_tx(struct channel *channel, struct bitcoin_tx *tx, - const struct bitcoin_signature *sig, - enum wallet_tx_type type); + const struct bitcoin_signature *sig); static inline bool channel_can_add_htlc(const struct channel *channel) { diff --git a/lightningd/channel_control.c b/lightningd/channel_control.c index 7bef2284f430..94dc7e4831fe 100644 --- a/lightningd/channel_control.c +++ b/lightningd/channel_control.c @@ -1067,10 +1067,8 @@ struct command_result *cancel_channel_before_broadcast(struct command *cmd, /* Check if we broadcast the transaction. (We store the transaction * type into DB before broadcast). */ - enum wallet_tx_type type; - if (wallet_transaction_type(cmd->ld->wallet, - &cancel_channel->funding.txid, - &type)) + if (wallet_transaction_get(tmpctx, cmd->ld->wallet, + &cancel_channel->funding.txid)) return command_fail(cmd, FUNDING_CANCEL_NOT_SAFE, "Has the funding transaction been" " broadcast? Please use `close` or" diff --git a/lightningd/closing_control.c b/lightningd/closing_control.c index 03b6565ad1b5..73ccaf835d56 100644 --- a/lightningd/closing_control.c +++ b/lightningd/closing_control.c @@ -266,7 +266,7 @@ static void peer_received_closing_signature(struct channel *channel, } if (closing_fee_is_acceptable(ld, channel, tx)) { - channel_set_last_tx(channel, tx, &sig, TX_CHANNEL_CLOSE); + channel_set_last_tx(channel, tx, &sig); wallet_channel_save(ld->wallet, channel); } diff --git a/lightningd/dual_open_control.c b/lightningd/dual_open_control.c index 428c1ede1984..1743186bfd9c 100644 --- a/lightningd/dual_open_control.c +++ b/lightningd/dual_open_control.c @@ -1149,8 +1149,7 @@ wallet_update_channel(struct lightningd *ld, channel_set_last_tx(channel, tal_steal(channel, remote_commit), - remote_commit_sig, - TX_CHANNEL_UNILATERAL); + remote_commit_sig); /* Update in database */ wallet_channel_save(ld->wallet, channel); @@ -1238,7 +1237,6 @@ wallet_commit_channel(struct lightningd *ld, channel->last_tx = tal_steal(channel, remote_commit); channel->last_sig = *remote_commit_sig; - channel->last_tx_type = TX_CHANNEL_UNILATERAL; channel->channel_info = *channel_info; channel->fee_states = new_fee_states(channel, diff --git a/lightningd/onchain_control.c b/lightningd/onchain_control.c index e881d7a77338..0a7e6f3ff746 100644 --- a/lightningd/onchain_control.c +++ b/lightningd/onchain_control.c @@ -329,20 +329,16 @@ static void handle_onchain_broadcast_tx(struct channel *channel, { struct bitcoin_tx *tx; struct wallet *w = channel->peer->ld->wallet; - struct bitcoin_txid txid; - enum wallet_tx_type type; bool is_rbf; - if (!fromwire_onchaind_broadcast_tx(msg, msg, &tx, &type, &is_rbf)) { + if (!fromwire_onchaind_broadcast_tx(msg, msg, &tx, &is_rbf)) { channel_internal_error(channel, "Invalid onchain_broadcast_tx"); return; } tx->chainparams = chainparams; - bitcoin_txid(tx, &txid); wallet_transaction_add(w, tx->wtx, 0, 0); - wallet_transaction_annotate(w, &txid, type, channel->dbid); /* We don't really care if it fails, we'll respond via watch. */ /* If the onchaind signals this as RBF-able, then we also diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index dfb75e795b05..a084ded4fb6b 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -283,9 +283,6 @@ static void sign_and_send_last(struct lightningd *ld, sign_last_tx(channel, last_tx, last_sig); bitcoin_txid(last_tx, &txid); wallet_transaction_add(ld->wallet, last_tx->wtx, 0, 0); - wallet_transaction_annotate(ld->wallet, &txid, - channel->last_tx_type, - channel->dbid); /* Keep broadcasting until we say stop (can fail due to dup, * if they beat us to the broadcast). */ @@ -1739,8 +1736,7 @@ static void update_channel_from_inflight(struct lightningd *ld, psbt_copy = clone_psbt(channel, inflight->last_tx->psbt); channel_set_last_tx(channel, bitcoin_tx_with_psbt(channel, psbt_copy), - &inflight->last_sig, - TX_CHANNEL_UNILATERAL); + &inflight->last_sig); /* Update the reserve */ channel_update_reserve(channel, diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 2bb9531fedf6..c62cb669a03c 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -1982,7 +1982,7 @@ static bool peer_save_commitsig_received(struct channel *channel, u64 commitnum, channel->next_index[LOCAL]++; /* Update channel->last_sig and channel->last_tx before saving to db */ - channel_set_last_tx(channel, tx, commit_sig, TX_CHANNEL_UNILATERAL); + channel_set_last_tx(channel, tx, commit_sig); return true; } diff --git a/lightningd/test/run-invoice-select-inchan.c b/lightningd/test/run-invoice-select-inchan.c index 93d624d18d03..2e265e827541 100644 --- a/lightningd/test/run-invoice-select-inchan.c +++ b/lightningd/test/run-invoice-select-inchan.c @@ -97,8 +97,7 @@ u32 channel_last_funding_feerate(const struct channel *channel UNNEEDED) /* Generated stub for channel_set_last_tx */ void channel_set_last_tx(struct channel *channel UNNEEDED, struct bitcoin_tx *tx UNNEEDED, - const struct bitcoin_signature *sig UNNEEDED, - enum wallet_tx_type type UNNEEDED) + const struct bitcoin_signature *sig UNNEEDED) { fprintf(stderr, "channel_set_last_tx called!\n"); abort(); } /* Generated stub for channel_state_name */ const char *channel_state_name(const struct channel *channel UNNEEDED) @@ -946,11 +945,6 @@ struct amount_msat wallet_total_forward_fees(struct wallet *w UNNEEDED) void wallet_transaction_add(struct wallet *w UNNEEDED, const struct wally_tx *tx UNNEEDED, const u32 blockheight UNNEEDED, const u32 txindex UNNEEDED) { fprintf(stderr, "wallet_transaction_add called!\n"); abort(); } -/* Generated stub for wallet_transaction_annotate */ -void wallet_transaction_annotate(struct wallet *w UNNEEDED, - const struct bitcoin_txid *txid UNNEEDED, - enum wallet_tx_type type UNNEEDED, u64 channel_id UNNEEDED) -{ fprintf(stderr, "wallet_transaction_annotate called!\n"); abort(); } /* Generated stub for wallet_transaction_locate */ struct txlocator *wallet_transaction_locate(const tal_t *ctx UNNEEDED, struct wallet *w UNNEEDED, const struct bitcoin_txid *txid UNNEEDED) diff --git a/onchaind/onchaind.c b/onchaind/onchaind.c index 8c9e971d7fd9..e440bddcce57 100644 --- a/onchaind/onchaind.c +++ b/onchaind/onchaind.c @@ -1018,43 +1018,6 @@ static void ignore_output(struct tracked_output *out) out->resolved->tx_type = SELF; } -static enum wallet_tx_type onchain_txtype_to_wallet_txtype(enum tx_type t) -{ - switch (t) { - case FUNDING_TRANSACTION: - return TX_CHANNEL_FUNDING; - case MUTUAL_CLOSE: - return TX_CHANNEL_CLOSE; - case OUR_UNILATERAL: - return TX_CHANNEL_UNILATERAL; - case THEIR_HTLC_FULFILL_TO_US: - case OUR_HTLC_SUCCESS_TX: - return TX_CHANNEL_HTLC_SUCCESS; - case OUR_HTLC_TIMEOUT_TO_US: - case OUR_HTLC_TIMEOUT_TX: - return TX_CHANNEL_HTLC_TIMEOUT; - case OUR_DELAYED_RETURN_TO_WALLET: - case SELF: - return TX_CHANNEL_SWEEP; - case OUR_PENALTY_TX: - return TX_CHANNEL_PENALTY; - case THEIR_DELAYED_CHEAT: - return TX_CHANNEL_CHEAT | TX_THEIRS; - case THEIR_UNILATERAL: - case UNKNOWN_UNILATERAL: - case THEIR_REVOKED_UNILATERAL: - return TX_CHANNEL_UNILATERAL | TX_THEIRS; - case THEIR_HTLC_TIMEOUT_TO_THEM: - return TX_CHANNEL_HTLC_TIMEOUT | TX_THEIRS; - case OUR_HTLC_FULFILL_TO_THEM: - return TX_CHANNEL_HTLC_SUCCESS | TX_THEIRS; - case IGNORING_TINY_PAYMENT: - case UNKNOWN_TXTYPE: - return TX_UNKNOWN; - } - abort(); -} - /** proposal_is_rbfable * * @brief returns true if the given proposal @@ -1141,8 +1104,6 @@ static void proposal_should_rbf(struct tracked_output *out) /* Broadcast the transaction. */ if (tx) { - enum wallet_tx_type wtt; - status_debug("Broadcasting RBF %s (%s) to resolve %s/%s " "depth=%"PRIu32"", tx_type_name(out->proposal->tx_type), @@ -1151,11 +1112,9 @@ static void proposal_should_rbf(struct tracked_output *out) output_type_name(out->output_type), depth); - wtt = onchain_txtype_to_wallet_txtype(out->proposal->tx_type); wire_sync_write(REQ_FD, take(towire_onchaind_broadcast_tx(NULL, tx, - wtt, - true))); + true))); } } @@ -1186,9 +1145,7 @@ static void proposal_meets_depth(struct tracked_output *out) wire_sync_write( REQ_FD, take(towire_onchaind_broadcast_tx( - NULL, out->proposal->tx, - onchain_txtype_to_wallet_txtype(out->proposal->tx_type), - is_rbf))); + NULL, out->proposal->tx, is_rbf))); /* Don't wait for this if we're ignoring the tiny payment. */ if (out->proposal->tx_type == IGNORING_TINY_PAYMENT) { diff --git a/onchaind/onchaind_wire.csv b/onchaind/onchaind_wire.csv index f6f3776a37c8..1f891b6639d6 100644 --- a/onchaind/onchaind_wire.csv +++ b/onchaind/onchaind_wire.csv @@ -73,7 +73,6 @@ msgdata,onchaind_htlcs,tell_immediately,bool,num_htlcs # it with a higher fee. msgtype,onchaind_broadcast_tx,5003 msgdata,onchaind_broadcast_tx,tx,bitcoin_tx, -msgdata,onchaind_broadcast_tx,type,enum wallet_tx_type, msgdata,onchaind_broadcast_tx,is_rbf,bool, # master->onchaind: Notifier that an output has been spent by input_num of tx. diff --git a/onchaind/test/run-grind_feerate-bug.c b/onchaind/test/run-grind_feerate-bug.c index 0c069752d0c9..e9d66915672a 100644 --- a/onchaind/test/run-grind_feerate-bug.c +++ b/onchaind/test/run-grind_feerate-bug.c @@ -262,7 +262,7 @@ u8 *towire_onchaind_annotate_txin(const tal_t *ctx UNNEEDED, const struct bitcoi u8 *towire_onchaind_annotate_txout(const tal_t *ctx UNNEEDED, const struct bitcoin_outpoint *outpoint UNNEEDED, enum wallet_tx_type type UNNEEDED) { fprintf(stderr, "towire_onchaind_annotate_txout called!\n"); abort(); } /* Generated stub for towire_onchaind_broadcast_tx */ -u8 *towire_onchaind_broadcast_tx(const tal_t *ctx UNNEEDED, const struct bitcoin_tx *tx UNNEEDED, enum wallet_tx_type type UNNEEDED, bool is_rbf UNNEEDED) +u8 *towire_onchaind_broadcast_tx(const tal_t *ctx UNNEEDED, const struct bitcoin_tx *tx UNNEEDED, bool is_rbf UNNEEDED) { fprintf(stderr, "towire_onchaind_broadcast_tx called!\n"); abort(); } /* Generated stub for towire_onchaind_dev_memleak_reply */ u8 *towire_onchaind_dev_memleak_reply(const tal_t *ctx UNNEEDED, bool leak UNNEEDED) diff --git a/onchaind/test/run-grind_feerate.c b/onchaind/test/run-grind_feerate.c index 7eb8d65166a6..99e8ec1495e8 100644 --- a/onchaind/test/run-grind_feerate.c +++ b/onchaind/test/run-grind_feerate.c @@ -288,7 +288,7 @@ u8 *towire_onchaind_annotate_txin(const tal_t *ctx UNNEEDED, const struct bitcoi u8 *towire_onchaind_annotate_txout(const tal_t *ctx UNNEEDED, const struct bitcoin_outpoint *outpoint UNNEEDED, enum wallet_tx_type type UNNEEDED) { fprintf(stderr, "towire_onchaind_annotate_txout called!\n"); abort(); } /* Generated stub for towire_onchaind_broadcast_tx */ -u8 *towire_onchaind_broadcast_tx(const tal_t *ctx UNNEEDED, const struct bitcoin_tx *tx UNNEEDED, enum wallet_tx_type type UNNEEDED, bool is_rbf UNNEEDED) +u8 *towire_onchaind_broadcast_tx(const tal_t *ctx UNNEEDED, const struct bitcoin_tx *tx UNNEEDED, bool is_rbf UNNEEDED) { fprintf(stderr, "towire_onchaind_broadcast_tx called!\n"); abort(); } /* Generated stub for towire_onchaind_dev_memleak_reply */ u8 *towire_onchaind_dev_memleak_reply(const tal_t *ctx UNNEEDED, bool leak UNNEEDED) diff --git a/wallet/db.c b/wallet/db.c index 4a45167f5fda..927e5dd117d9 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -492,6 +492,8 @@ static struct migration dbmigrations[] = { /* remote signatures for channel announcement */ {SQL("ALTER TABLE channels ADD remote_ann_node_sig BLOB;"), NULL}, {SQL("ALTER TABLE channels ADD remote_ann_bitcoin_sig BLOB;"), NULL}, + /* FIXME: We now use the transaction_annotations table to type each + * input and output instead of type and channel_id! */ /* Additional information for transaction tracking and listing */ {SQL("ALTER TABLE transactions ADD type BIGINT;"), NULL}, /* Not a foreign key on purpose since we still delete channels from diff --git a/wallet/wallet.c b/wallet/wallet.c index 1424dfc3bbc1..6bfed095d0ea 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -4133,66 +4133,6 @@ void wallet_annotate_txin(struct wallet *w, const struct bitcoin_txid *txid, wallet_annotation_add(w, txid, innum, INPUT_ANNOTATION, type, channel); } -void wallet_transaction_annotate(struct wallet *w, - const struct bitcoin_txid *txid, enum wallet_tx_type type, - u64 channel_id) -{ - struct db_stmt *stmt = db_prepare_v2( - w->db, SQL("SELECT type, channel_id FROM transactions WHERE id=?")); - db_bind_txid(stmt, 0, txid); - db_query_prepared(stmt); - - if (!db_step(stmt)) - fatal("Attempting to annotate a transaction we don't have: %s", - type_to_string(tmpctx, struct bitcoin_txid, txid)); - - if (!db_col_is_null(stmt, "type")) - type |= db_col_u64(stmt, "type"); - - if (channel_id == 0 && !db_col_is_null(stmt, "channel_id")) - channel_id = db_col_u64(stmt, "channel_id"); - else - db_col_ignore(stmt, "channel_id"); - - tal_free(stmt); - - stmt = db_prepare_v2(w->db, SQL("UPDATE transactions " - "SET type = ?" - ", channel_id = ? " - "WHERE id = ?")); - - db_bind_u64(stmt, 0, type); - - if (channel_id) - db_bind_int(stmt, 1, channel_id); - else - db_bind_null(stmt, 1); - - db_bind_txid(stmt, 2, txid); - db_exec_prepared_v2(take(stmt)); -} - -bool wallet_transaction_type(struct wallet *w, const struct bitcoin_txid *txid, - enum wallet_tx_type *type) -{ - struct db_stmt *stmt = db_prepare_v2(w->db, SQL("SELECT type FROM transactions WHERE id=?")); - db_bind_sha256(stmt, 0, &txid->shad.sha); - db_query_prepared(stmt); - - if (!db_step(stmt)) { - tal_free(stmt); - return false; - } - - if (!db_col_is_null(stmt, "type")) - *type = db_col_u64(stmt, "type"); - else - *type = 0; - - tal_free(stmt); - return true; -} - struct bitcoin_tx *wallet_transaction_get(const tal_t *ctx, struct wallet *w, const struct bitcoin_txid *txid) { @@ -4794,8 +4734,6 @@ struct wallet_transaction *wallet_transactions_get(struct wallet *w, const tal_t ", t.rawtx" ", t.blockheight" ", t.txindex" - ", t.type as txtype" - ", c2.scid as txchan" ", a.location" ", a.idx as ann_idx" ", a.type as annotation_type" @@ -4803,8 +4741,7 @@ struct wallet_transaction *wallet_transactions_get(struct wallet *w, const tal_t " FROM" " transactions t LEFT JOIN" " transaction_annotations a ON (a.txid = t.id) LEFT JOIN" - " channels c ON (a.channel = c.id) LEFT JOIN" - " channels c2 ON (t.channel_id = c2.id) " + " channels c ON (a.channel = c.id) " "ORDER BY t.blockheight, t.txindex ASC")); db_query_prepared(stmt); @@ -4836,16 +4773,6 @@ struct wallet_transaction *wallet_transactions_get(struct wallet *w, const tal_t cur->blockheight = 0; cur->txindex = 0; } - if (!db_col_is_null(stmt, "txtype")) - cur->annotation.type - = db_col_u64(stmt, "txtype"); - else - cur->annotation.type = 0; - if (!db_col_is_null(stmt, "txchan")) - db_col_scid(stmt, "txchan", &cur->annotation.channel); - else - cur->annotation.channel.u64 = 0; - cur->output_annotations = tal_arrz(txs, struct tx_annotation, cur->tx->wtx->num_outputs); cur->input_annotations = tal_arrz(txs, struct tx_annotation, cur->tx->wtx->num_inputs); } diff --git a/wallet/wallet.h b/wallet/wallet.h index fe84c6ffeae1..537580a3dba6 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -408,8 +408,6 @@ struct wallet_transaction { /* Fully parsed transaction */ const struct bitcoin_tx *tx; - struct tx_annotation annotation; - /* tal_arr containing the annotation types, if any, for the respective * inputs and outputs. 0 if there are no annotations for the * element. */ @@ -1291,28 +1289,6 @@ void wallet_annotate_txout(struct wallet *w, void wallet_annotate_txin(struct wallet *w, const struct bitcoin_txid *txid, int innum, enum wallet_tx_type type, u64 channel); -/** - * Annotate a transaction in the DB with its type and channel referemce. - * - * We add transactions when filtering the block, but often know its type only - * when we trigger the txwatches, at which point we've already discarded the - * full transaction. This function can be used to annotate the transactions - * after the fact with a channel number for grouping and a type for filtering. - */ -void wallet_transaction_annotate(struct wallet *w, - const struct bitcoin_txid *txid, - enum wallet_tx_type type, u64 channel_id); - -/** - * Get the type of a transaction we are watching by its - * txid. - * - * Returns false if the transaction was not stored in DB. - * Returns true if the transaction exists and sets the `type` parameter. - */ -bool wallet_transaction_type(struct wallet *w, const struct bitcoin_txid *txid, - enum wallet_tx_type *type); - /** * Get the transaction from the database * diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index 4292781acd24..9c9dc22ee22c 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -1,6 +1,7 @@ #include "config.h" #include #include +#include #include #include #include @@ -483,28 +484,18 @@ struct { {TX_CHANNEL_HTLC_TIMEOUT, "channel_htlc_timeout"}, {TX_CHANNEL_PENALTY, "channel_penalty"}, {TX_CHANNEL_CHEAT, "channel_unilateral_cheat"}, - {0, NULL} }; #if EXPERIMENTAL_FEATURES static const char *txtype_to_string(enum wallet_tx_type t) { - for (size_t i = 0; wallet_tx_type_display_names[i].name != NULL; i++) + for (size_t i = 0; i < ARRAY_SIZE(wallet_tx_type_display_names); i++) if (t == wallet_tx_type_display_names[i].t) return wallet_tx_type_display_names[i].name; return NULL; } - -static void json_add_txtypes(struct json_stream *result, const char *fieldname, enum wallet_tx_type value) -{ - json_array_start(result, fieldname); - for (size_t i = 0; wallet_tx_type_display_names[i].name != NULL; i++) { - if (value & wallet_tx_type_display_names[i].t) - json_add_string(result, NULL, wallet_tx_type_display_names[i].name); - } - json_array_end(result); -} #endif + static void json_transaction_details(struct json_stream *response, const struct wallet_transaction *tx) { @@ -515,13 +506,6 @@ static void json_transaction_details(struct json_stream *response, json_add_hex_talarr(response, "rawtx", tx->rawtx); json_add_num(response, "blockheight", tx->blockheight); json_add_num(response, "txindex", tx->txindex); -#if EXPERIMENTAL_FEATURES - if (tx->annotation.type != 0) - json_add_txtypes(response, "type", tx->annotation.type); - - if (tx->annotation.channel.u64 != 0) - json_add_short_channel_id(response, "channel", &tx->annotation.channel); -#endif json_add_u32(response, "locktime", wtx->locktime); json_add_u32(response, "version", wtx->version); From a880b60934cf4654b0d1d70b6baf92c111edea70 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:37:03 +1030 Subject: [PATCH 10/36] plugins/topology: add direction field to listchannels. It's a core concept in the spec which isn't directly exposed. Signed-off-by: Rusty Russell Changelog-Added: JSON-RPC: `listchannels` added a `direction` field (0 or 1) as per gossip specification. --- .msggen.json | 1 + cln-grpc/proto/node.proto | 1 + cln-grpc/src/convert.rs | 1 + cln-rpc/src/model.rs | 1 + contrib/pyln-testing/pyln/testing/grpc2py.py | 1 + doc/lightning-listchannels.7.md | 3 ++- doc/schemas/listchannels.schema.json | 5 +++++ plugins/topology.c | 1 + tests/test_gossip.py | 8 ++++++++ 9 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.msggen.json b/.msggen.json index 4bb7a19d7f1e..1178f8385c4e 100644 --- a/.msggen.json +++ b/.msggen.json @@ -545,6 +545,7 @@ "ListChannels.channels[].channel_flags": 7, "ListChannels.channels[].delay": 12, "ListChannels.channels[].destination": 2, + "ListChannels.channels[].direction": 16, "ListChannels.channels[].features": 15, "ListChannels.channels[].fee_per_millionth": 11, "ListChannels.channels[].htlc_maximum_msat": 14, diff --git a/cln-grpc/proto/node.proto b/cln-grpc/proto/node.proto index b7386cb25ed1..c15e8c7f2dbb 100644 --- a/cln-grpc/proto/node.proto +++ b/cln-grpc/proto/node.proto @@ -362,6 +362,7 @@ message ListchannelsChannels { bytes source = 1; bytes destination = 2; string short_channel_id = 3; + uint32 direction = 16; bool public = 4; Amount amount_msat = 5; uint32 message_flags = 6; diff --git a/cln-grpc/src/convert.rs b/cln-grpc/src/convert.rs index df5bb8e7d0bd..9e8cd80a4ee4 100644 --- a/cln-grpc/src/convert.rs +++ b/cln-grpc/src/convert.rs @@ -306,6 +306,7 @@ impl From for pb::ListchannelsChannels { source: c.source.serialize().to_vec(), // Rule #2 for type pubkey destination: c.destination.serialize().to_vec(), // Rule #2 for type pubkey short_channel_id: c.short_channel_id.to_string(), // Rule #2 for type short_channel_id + direction: c.direction, // Rule #2 for type u32 public: c.public, // Rule #2 for type boolean amount_msat: Some(c.amount_msat.into()), // Rule #2 for type msat message_flags: c.message_flags.into(), // Rule #2 for type u8 diff --git a/cln-rpc/src/model.rs b/cln-rpc/src/model.rs index 45ab1e82ea5d..5be4b8e879d3 100644 --- a/cln-rpc/src/model.rs +++ b/cln-rpc/src/model.rs @@ -1869,6 +1869,7 @@ pub mod responses { pub source: PublicKey, pub destination: PublicKey, pub short_channel_id: ShortChannelId, + pub direction: u32, pub public: bool, pub amount_msat: Amount, pub message_flags: u8, diff --git a/contrib/pyln-testing/pyln/testing/grpc2py.py b/contrib/pyln-testing/pyln/testing/grpc2py.py index 5a4bdc1a35f3..3e8e2eb8f488 100644 --- a/contrib/pyln-testing/pyln/testing/grpc2py.py +++ b/contrib/pyln-testing/pyln/testing/grpc2py.py @@ -258,6 +258,7 @@ def listchannels_channels2py(m): "source": hexlify(m.source), # PrimitiveField in generate_composite "destination": hexlify(m.destination), # PrimitiveField in generate_composite "short_channel_id": m.short_channel_id, # PrimitiveField in generate_composite + "direction": m.direction, # PrimitiveField in generate_composite "public": m.public, # PrimitiveField in generate_composite "amount_msat": amount2msat(m.amount_msat), # PrimitiveField in generate_composite "message_flags": m.message_flags, # PrimitiveField in generate_composite diff --git a/doc/lightning-listchannels.7.md b/doc/lightning-listchannels.7.md index 877fd440fc37..96e7cc7de227 100644 --- a/doc/lightning-listchannels.7.md +++ b/doc/lightning-listchannels.7.md @@ -36,6 +36,7 @@ On success, an object containing **channels** is returned. It is an array of ob - **source** (pubkey): the source node - **destination** (pubkey): the destination node - **short\_channel\_id** (short\_channel\_id): short channel id of channel +- **direction** (u32): direction (0 if source < destination, 1 otherwise). - **public** (boolean): true if this is announced (otherwise it must be our channel) - **amount\_msat** (msat): the total capacity of this channel (always a whole number of satoshis) - **message\_flags** (u8): as defined by BOLT #7 @@ -79,4 +80,4 @@ Lightning RFC site - BOLT \#7: -[comment]: # ( SHA256STAMP:d8d52272963a9ec4708fd3ae41585ddd8120bb5444c03219a7b728f0b2e09ec3) +[comment]: # ( SHA256STAMP:78f59780528ae5cd33c3607ed11b128cc94e1e0a5e2babd59cb99574a3f5f956) diff --git a/doc/schemas/listchannels.schema.json b/doc/schemas/listchannels.schema.json index feea2028531d..acc08380647b 100644 --- a/doc/schemas/listchannels.schema.json +++ b/doc/schemas/listchannels.schema.json @@ -15,6 +15,7 @@ "source", "destination", "short_channel_id", + "direction", "public", "amount_msat", "message_flags", @@ -40,6 +41,10 @@ "type": "short_channel_id", "description": "short channel id of channel" }, + "direction": { + "type": "u32", + "description": "direction (0 if source < destination, 1 otherwise)." + }, "public": { "type": "boolean", "description": "true if this is announced (otherwise it must be our channel)" diff --git a/plugins/topology.c b/plugins/topology.c index 943dcd54d0c3..a8e6017a703a 100644 --- a/plugins/topology.c +++ b/plugins/topology.c @@ -244,6 +244,7 @@ static void json_add_halfchan(struct json_stream *response, json_add_node_id(response, "source", &node_id[dir]); json_add_node_id(response, "destination", &node_id[!dir]); json_add_short_channel_id(response, "short_channel_id", &scid); + json_add_num(response, "direction", dir); json_add_bool(response, "public", !c->private); gossmap_chan_get_update_details(gossmap, c, dir, diff --git a/tests/test_gossip.py b/tests/test_gossip.py index f4c5b52de02c..c7acc448190f 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -425,6 +425,10 @@ def test_gossip_jsonrpc(node_factory): channels2 = l2.rpc.listchannels(source=l1.info['id'])['channels'] assert only_one(channels1)['source'] == l1.info['id'] assert only_one(channels1)['destination'] == l2.info['id'] + if l1.info['id'] > l2.info['id']: + assert only_one(channels1)['direction'] == 1 + else: + assert only_one(channels1)['direction'] == 0 assert channels1 == channels2 # Test listchannels-by-destination @@ -432,6 +436,10 @@ def test_gossip_jsonrpc(node_factory): channels2 = l2.rpc.listchannels(destination=l1.info['id'])['channels'] assert only_one(channels1)['destination'] == l1.info['id'] assert only_one(channels1)['source'] == l2.info['id'] + if l2.info['id'] > l1.info['id']: + assert only_one(channels1)['direction'] == 1 + else: + assert only_one(channels1)['direction'] == 0 assert channels1 == channels2 # Test only one of short_channel_id, source or destination can be supplied From 90bb11dac9ef0094be8034de124dc6f5aa1212c3 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:38:03 +1030 Subject: [PATCH 11/36] lightningd: fix type of listhtlcs payment_hash. `hash` is a tighter requirement than simply `hex`. Signed-off-by: Rusty Russell --- doc/lightning-listhtlcs.7.md | 4 ++-- doc/schemas/listhtlcs.schema.json | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/lightning-listhtlcs.7.md b/doc/lightning-listhtlcs.7.md index 8200b5ac306a..82d0019a01a4 100644 --- a/doc/lightning-listhtlcs.7.md +++ b/doc/lightning-listhtlcs.7.md @@ -26,7 +26,7 @@ On success, an object containing **htlcs** is returned. It is an array of objec - **expiry** (u32): the block number where this HTLC expires/expired - **amount\_msat** (msat): the value of the HTLC - **direction** (string): out if we offered this to the peer, in if they offered it (one of "out", "in") -- **payment\_hash** (hex): payment hash sought by HTLC (always 64 characters) +- **payment\_hash** (hash): payment hash sought by HTLC - **state** (string): The first 10 states are for `in`, the next 10 are for `out`. (one of "SENT\_ADD\_HTLC", "SENT\_ADD\_COMMIT", "RCVD\_ADD\_REVOCATION", "RCVD\_ADD\_ACK\_COMMIT", "SENT\_ADD\_ACK\_REVOCATION", "RCVD\_REMOVE\_HTLC", "RCVD\_REMOVE\_COMMIT", "SENT\_REMOVE\_REVOCATION", "SENT\_REMOVE\_ACK\_COMMIT", "RCVD\_REMOVE\_ACK\_REVOCATION", "RCVD\_ADD\_HTLC", "RCVD\_ADD\_COMMIT", "SENT\_ADD\_REVOCATION", "SENT\_ADD\_ACK\_COMMIT", "RCVD\_ADD\_ACK\_REVOCATION", "SENT\_REMOVE\_HTLC", "SENT\_REMOVE\_COMMIT", "RCVD\_REMOVE\_REVOCATION", "RCVD\_REMOVE\_ACK\_COMMIT", "SENT\_REMOVE\_ACK\_REVOCATION") [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -46,4 +46,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:2f658fb394c49408c60c03b396bbc5afe7465fd33f48d8043233f2fe2b76f25e) +[comment]: # ( SHA256STAMP:990e36b109c9e318bc566a951ce0d39032e252cdd1555c75ad7b168d547c937f) diff --git a/doc/schemas/listhtlcs.schema.json b/doc/schemas/listhtlcs.schema.json index 469eb1588bbe..8de6f5462a88 100644 --- a/doc/schemas/listhtlcs.schema.json +++ b/doc/schemas/listhtlcs.schema.json @@ -46,10 +46,8 @@ "description": "out if we offered this to the peer, in if they offered it" }, "payment_hash": { - "type": "hex", - "description": "payment hash sought by HTLC", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "payment hash sought by HTLC" }, "state": { "type": "string", From c74bc54598c25f920c1c2b0c25af34a51d214904 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:39:03 +1030 Subject: [PATCH 12/36] doc: fix listsendpays man page. We actually had a partid allowed (in the oneOf clauses), but didn''t document it. Signed-off-by: Rusty Russell --- .msggen.json | 1 + cln-grpc/proto/node.proto | 1 + cln-grpc/src/convert.rs | 1 + cln-rpc/src/model.rs | 2 ++ contrib/pyln-testing/pyln/testing/grpc2py.py | 1 + doc/lightning-listsendpays.7.md | 3 ++- doc/schemas/listsendpays.schema.json | 4 ++++ 7 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.msggen.json b/.msggen.json index 1178f8385c4e..311d64ee04d9 100644 --- a/.msggen.json +++ b/.msggen.json @@ -819,6 +819,7 @@ "ListSendPays.payments[].groupid": 2, "ListSendPays.payments[].id": 1, "ListSendPays.payments[].label": 9, + "ListSendPays.payments[].partid": 15, "ListSendPays.payments[].payment_hash": 3, "ListSendPays.payments[].payment_preimage": 12, "ListSendPays.payments[].status": 4 diff --git a/cln-grpc/proto/node.proto b/cln-grpc/proto/node.proto index c15e8c7f2dbb..981ae294649f 100644 --- a/cln-grpc/proto/node.proto +++ b/cln-grpc/proto/node.proto @@ -719,6 +719,7 @@ message ListsendpaysPayments { } uint64 id = 1; uint64 groupid = 2; + optional uint64 partid = 15; bytes payment_hash = 3; ListsendpaysPaymentsStatus status = 4; optional Amount amount_msat = 5; diff --git a/cln-grpc/src/convert.rs b/cln-grpc/src/convert.rs index 9e8cd80a4ee4..6b8032f5ad2d 100644 --- a/cln-grpc/src/convert.rs +++ b/cln-grpc/src/convert.rs @@ -574,6 +574,7 @@ impl From for pb::ListsendpaysPayments { Self { id: c.id, // Rule #2 for type u64 groupid: c.groupid, // Rule #2 for type u64 + partid: c.partid, // Rule #2 for type u64? payment_hash: c.payment_hash.to_vec(), // Rule #2 for type hash status: c.status as i32, amount_msat: c.amount_msat.map(|f| f.into()), // Rule #2 for type msat? diff --git a/cln-rpc/src/model.rs b/cln-rpc/src/model.rs index 5be4b8e879d3..cca63253dafc 100644 --- a/cln-rpc/src/model.rs +++ b/cln-rpc/src/model.rs @@ -2473,6 +2473,8 @@ pub mod responses { pub struct ListsendpaysPayments { pub id: u64, pub groupid: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub partid: Option, pub payment_hash: Sha256, // Path `ListSendPays.payments[].status` pub status: ListsendpaysPaymentsStatus, diff --git a/contrib/pyln-testing/pyln/testing/grpc2py.py b/contrib/pyln-testing/pyln/testing/grpc2py.py index 3e8e2eb8f488..28b366c8bdda 100644 --- a/contrib/pyln-testing/pyln/testing/grpc2py.py +++ b/contrib/pyln-testing/pyln/testing/grpc2py.py @@ -465,6 +465,7 @@ def listsendpays_payments2py(m): return remove_default({ "id": m.id, # PrimitiveField in generate_composite "groupid": m.groupid, # PrimitiveField in generate_composite + "partid": m.partid, # PrimitiveField in generate_composite "payment_hash": hexlify(m.payment_hash), # PrimitiveField in generate_composite "status": str(m.status), # EnumField in generate_composite "amount_msat": amount2msat(m.amount_msat), # PrimitiveField in generate_composite diff --git a/doc/lightning-listsendpays.7.md b/doc/lightning-listsendpays.7.md index 1d734b40c28f..52ddbd108db8 100644 --- a/doc/lightning-listsendpays.7.md +++ b/doc/lightning-listsendpays.7.md @@ -31,6 +31,7 @@ On success, an object containing **payments** is returned. It is an array of ob - **status** (string): status of the payment (one of "pending", "failed", "complete") - **created\_at** (u64): the UNIX timestamp showing when this payment was initiated - **amount\_sent\_msat** (msat): The amount sent +- **partid** (u64, optional): Part number (for multiple parts to a single payment) - **amount\_msat** (msat, optional): The amount delivered to destination (if known) - **destination** (pubkey, optional): the final destination of the payment if known - **label** (string, optional): the label, if given to sendpay @@ -64,4 +65,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:c7a067147e3275afa7f0cad68a6e1d9c0a10fad038a7b95b9c173edf523aee23) +[comment]: # ( SHA256STAMP:a6dcb3708d706650f74fcdd4d05614a813ac5a69c13c4c579d45c01b106545e2) diff --git a/doc/schemas/listsendpays.schema.json b/doc/schemas/listsendpays.schema.json index 3c57246e2776..ed0cb013ed99 100644 --- a/doc/schemas/listsendpays.schema.json +++ b/doc/schemas/listsendpays.schema.json @@ -28,6 +28,10 @@ "type": "u64", "description": "Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash" }, + "partid": { + "type": "u64", + "description": "Part number (for multiple parts to a single payment)" + }, "payment_hash": { "type": "hash", "description": "the hash of the *payment_preimage* which will prove payment", From c82651319ffd24150cde3a719a1154fe00139bda Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:08 +1030 Subject: [PATCH 13/36] common/gossip_store: clean up header. It's actually two separate u16 fields, so actually treat it as such! Cleans up zombie handling code a bit too. Signed-off-by: Rusty Russell --- common/gossip_store.c | 28 +++--- common/gossip_store.h | 27 ++---- common/gossmap.c | 23 ++--- common/test/run-route-specific.c | 3 +- common/test/run-route.c | 3 +- devtools/dump-gossipstore.c | 12 +-- gossipd/gossip_store.c | 153 ++++++++++++------------------ plugins/test/run-route-overlong.c | 3 +- 8 files changed, 106 insertions(+), 146 deletions(-) diff --git a/common/gossip_store.c b/common/gossip_store.c index 8f47b3b5f1a5..cf7aaaae8521 100644 --- a/common/gossip_store.c +++ b/common/gossip_store.c @@ -118,7 +118,8 @@ u8 *gossip_store_next(const tal_t *ctx, while (!msg) { struct gossip_hdr hdr; - u32 msglen, checksum, timestamp; + u16 msglen, flags; + u32 checksum, timestamp; bool push, ratelimited; int type, r; @@ -126,13 +127,13 @@ u8 *gossip_store_next(const tal_t *ctx, if (r != sizeof(hdr)) return NULL; - msglen = be32_to_cpu(hdr.len); - push = (msglen & GOSSIP_STORE_LEN_PUSH_BIT); - ratelimited = (msglen & GOSSIP_STORE_LEN_RATELIMIT_BIT); - msglen &= GOSSIP_STORE_LEN_MASK; + msglen = be16_to_cpu(hdr.len); + flags = be16_to_cpu(hdr.flags); + push = (flags & GOSSIP_STORE_PUSH_BIT); + ratelimited = (flags & GOSSIP_STORE_RATELIMIT_BIT); /* Skip any deleted entries. */ - if (be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_DELETED_BIT) { + if (flags & GOSSIP_STORE_DELETED_BIT) { *off += r + msglen; continue; } @@ -146,14 +147,6 @@ u8 *gossip_store_next(const tal_t *ctx, continue; } - /* Messages can be up to 64k, but we also have internal ones: - * 128k is plenty. */ - if (msglen > 128 * 1024) - status_failed(STATUS_FAIL_INTERNAL_ERROR, - "gossip_store: oversize msg len %u at" - " offset %zu (was at %zu)", - msglen, *off, initial_off); - checksum = be32_to_cpu(hdr.crc); msg = tal_arr(ctx, u8, msglen); r = pread(*gossip_store_fd, msg, msglen, *off + r); @@ -201,7 +194,7 @@ size_t find_gossip_store_end(int gossip_store_fd, size_t off) while ((r = pread(gossip_store_fd, &buf, sizeof(buf.hdr) + sizeof(buf.type), off)) == sizeof(buf.hdr) + sizeof(buf.type)) { - u32 msglen = be32_to_cpu(buf.hdr.len) & GOSSIP_STORE_LEN_MASK; + u16 msglen = be16_to_cpu(buf.hdr.len); /* Don't swallow end marker! */ if (buf.type == CPU_TO_BE16(WIRE_GOSSIP_STORE_ENDED)) @@ -227,7 +220,8 @@ size_t find_gossip_store_by_timestamp(int gossip_store_fd, while ((r = pread(gossip_store_fd, &buf, sizeof(buf.hdr) + sizeof(buf.type), off)) == sizeof(buf.hdr) + sizeof(buf.type)) { - u32 msglen = be32_to_cpu(buf.hdr.len) & GOSSIP_STORE_LEN_MASK; + u16 msglen = be16_to_cpu(buf.hdr.len); + u16 flags = be16_to_cpu(buf.hdr.flags); u16 type = be16_to_cpu(buf.type); /* Don't swallow end marker! Reset, as they will call @@ -236,7 +230,7 @@ size_t find_gossip_store_by_timestamp(int gossip_store_fd, return 1; /* Only to-be-broadcast types have valid timestamps! */ - if (!(be32_to_cpu(buf.hdr.len) & GOSSIP_STORE_LEN_DELETED_BIT) + if (!(flags & GOSSIP_STORE_DELETED_BIT) && public_msg_type(type) && be32_to_cpu(buf.hdr.timestamp) >= timestamp) { break; diff --git a/common/gossip_store.h b/common/gossip_store.h index 4ddf4289eda9..6ef02e3c8641 100644 --- a/common/gossip_store.h +++ b/common/gossip_store.h @@ -25,39 +25,32 @@ struct gossip_rcvd_filter; #define GOSSIP_STORE_MINOR_VERSION(verbyte) ((verbyte) & GOSSIP_STORE_MINOR_VERSION_MASK) /** - * Bit of length we use to mark a deleted record. + * Bit of flags we use to mark a deleted record. */ -#define GOSSIP_STORE_LEN_DELETED_BIT 0x80000000U +#define GOSSIP_STORE_DELETED_BIT 0x8000U /** - * Bit of length we use to mark an important record. + * Bit of flags we use to mark an important record. */ -#define GOSSIP_STORE_LEN_PUSH_BIT 0x40000000U +#define GOSSIP_STORE_PUSH_BIT 0x4000U /** - * Bit of length used to define a rate-limited record (do not rebroadcast) + * Bit of flags used to define a rate-limited record (do not rebroadcast) */ -#define GOSSIP_STORE_LEN_RATELIMIT_BIT 0x20000000U +#define GOSSIP_STORE_RATELIMIT_BIT 0x2000U /** - * Bit used to mark a channel announcement as inactive (needs channel updates.) + * Bit of flags used to mark a channel announcement as inactive (needs channel updates.) */ -#define GOSSIP_STORE_LEN_ZOMBIE_BIT 0x10000000U +#define GOSSIP_STORE_ZOMBIE_BIT 0x1000U -/** - * Full flags mask - */ -#define GOSSIP_STORE_FLAGS_MASK 0xFFFF0000U - -/* Mask for extracting just the length part of len field */ -#define GOSSIP_STORE_LEN_MASK \ - (~(GOSSIP_STORE_FLAGS_MASK)) /** * gossip_hdr -- On-disk format header. */ struct gossip_hdr { - beint32_t len; /* Length of message after header. */ + beint16_t flags; /* Length of message after header. */ + beint16_t len; /* Length of message after header. */ beint32_t crc; /* crc of message of timestamp, after header. */ beint32_t timestamp; /* timestamp of msg. */ }; diff --git a/common/gossmap.c b/common/gossmap.c index 31176ccf8d49..c7e2d348b678 100644 --- a/common/gossmap.c +++ b/common/gossmap.c @@ -611,16 +611,16 @@ static bool map_catchup(struct gossmap *map, size_t *num_rejected) map->map_end += reclen) { struct gossip_hdr ghdr; size_t off; - u16 type; + u16 type, flags; map_copy(map, map->map_end, &ghdr, sizeof(ghdr)); - reclen = (be32_to_cpu(ghdr.len) & GOSSIP_STORE_LEN_MASK) - + sizeof(ghdr); + reclen = be16_to_cpu(ghdr.len) + sizeof(ghdr); - if (be32_to_cpu(ghdr.len) & GOSSIP_STORE_LEN_DELETED_BIT) + flags = be16_to_cpu(ghdr.flags); + if (flags & GOSSIP_STORE_DELETED_BIT) continue; - if (be32_to_cpu(ghdr.len) & GOSSIP_STORE_LEN_ZOMBIE_BIT) + if (flags & GOSSIP_STORE_ZOMBIE_BIT) continue; /* Partial write, this can happen. */ @@ -1014,7 +1014,7 @@ bool gossmap_chan_get_capacity(const struct gossmap *map, /* Skip over this record to next; expect a gossip_store_channel_amount */ off = c->cann_off - sizeof(ghdr); map_copy(map, off, &ghdr, sizeof(ghdr)); - off += sizeof(ghdr) + (be32_to_cpu(ghdr.len) & GOSSIP_STORE_LEN_MASK); + off += sizeof(ghdr) + be16_to_cpu(ghdr.len); /* Partial write, this can happen. */ if (off + sizeof(ghdr) + 2 > map->map_size) @@ -1128,7 +1128,7 @@ u8 *gossmap_chan_get_announce(const tal_t *ctx, const struct gossmap *map, const struct gossmap_chan *c) { - u32 len; + u16 len; u8 *msg; u32 pre_off; @@ -1137,7 +1137,8 @@ u8 *gossmap_chan_get_announce(const tal_t *ctx, pre_off = 2 + 8 + 2 + sizeof(struct gossip_hdr); else pre_off = sizeof(struct gossip_hdr); - len = (map_be32(map, c->cann_off - pre_off) & GOSSIP_STORE_LEN_MASK); + len = map_be16(map, c->cann_off - pre_off + + offsetof(struct gossip_hdr, len)); msg = tal_arr(ctx, u8, len); map_copy(map, c->cann_off, msg, len); @@ -1149,14 +1150,14 @@ u8 *gossmap_node_get_announce(const tal_t *ctx, const struct gossmap *map, const struct gossmap_node *n) { - u32 len; + u16 len; u8 *msg; if (n->nann_off == 0) return NULL; - len = (map_be32(map, n->nann_off - sizeof(struct gossip_hdr)) - & GOSSIP_STORE_LEN_MASK); + len = map_be16(map, n->nann_off - sizeof(struct gossip_hdr) + + offsetof(struct gossip_hdr, len)); msg = tal_arr(ctx, u8, len); map_copy(map, n->nann_off, msg, len); diff --git a/common/test/run-route-specific.c b/common/test/run-route-specific.c index eddf83ba4d06..da067cc40230 100644 --- a/common/test/run-route-specific.c +++ b/common/test/run-route-specific.c @@ -51,7 +51,8 @@ static void write_to_store(int store_fd, const u8 *msg) { struct gossip_hdr hdr; - hdr.len = cpu_to_be32(tal_count(msg)); + hdr.flags = cpu_to_be16(0); + hdr.len = cpu_to_be16(tal_count(msg)); /* We don't actually check these! */ hdr.crc = 0; hdr.timestamp = 0; diff --git a/common/test/run-route.c b/common/test/run-route.c index e41b141264dc..b5d648dbdf5b 100644 --- a/common/test/run-route.c +++ b/common/test/run-route.c @@ -44,7 +44,8 @@ static void write_to_store(int store_fd, const u8 *msg) { struct gossip_hdr hdr; - hdr.len = cpu_to_be32(tal_count(msg)); + hdr.flags = cpu_to_be16(0); + hdr.len = cpu_to_be16(tal_count(msg)); /* We don't actually check these! */ hdr.crc = 0; hdr.timestamp = 0; diff --git a/devtools/dump-gossipstore.c b/devtools/dump-gossipstore.c index 80a9fef823d0..af0da532c067 100644 --- a/devtools/dump-gossipstore.c +++ b/devtools/dump-gossipstore.c @@ -65,17 +65,17 @@ int main(int argc, char *argv[]) while (read(fd, &hdr, sizeof(hdr)) == sizeof(hdr)) { struct amount_sat sat; struct short_channel_id scid; - u32 msglen = be32_to_cpu(hdr.len); + u16 flags = be16_to_cpu(hdr.flags); + u16 msglen = be16_to_cpu(hdr.len); u8 *msg, *inner; bool deleted, push, ratelimit, zombie; u32 blockheight; - deleted = (msglen & GOSSIP_STORE_LEN_DELETED_BIT); - push = (msglen & GOSSIP_STORE_LEN_PUSH_BIT); - ratelimit = (msglen & GOSSIP_STORE_LEN_RATELIMIT_BIT); - zombie = (msglen & GOSSIP_STORE_LEN_ZOMBIE_BIT); + deleted = (flags & GOSSIP_STORE_DELETED_BIT); + push = (flags & GOSSIP_STORE_PUSH_BIT); + ratelimit = (flags & GOSSIP_STORE_RATELIMIT_BIT); + zombie = (msglen & GOSSIP_STORE_ZOMBIE_BIT); - msglen &= GOSSIP_STORE_LEN_MASK; msg = tal_arr(NULL, u8, msglen); if (read(fd, msg, msglen) != msglen) errx(1, "%zu: Truncated file?", off); diff --git a/gossipd/gossip_store.c b/gossipd/gossip_store.c index a8a3b81a5483..1d4d6709316c 100644 --- a/gossipd/gossip_store.c +++ b/gossipd/gossip_store.c @@ -83,13 +83,14 @@ static bool append_msg(int fd, const u8 *msg, u32 timestamp, assert(*len); msglen = tal_count(msg); - hdr.len = cpu_to_be32(msglen); + hdr.len = cpu_to_be16(msglen); + hdr.flags = 0; if (push) - hdr.len |= CPU_TO_BE32(GOSSIP_STORE_LEN_PUSH_BIT); + hdr.flags |= CPU_TO_BE16(GOSSIP_STORE_PUSH_BIT); if (spam) - hdr.len |= CPU_TO_BE32(GOSSIP_STORE_LEN_RATELIMIT_BIT); + hdr.flags |= CPU_TO_BE16(GOSSIP_STORE_RATELIMIT_BIT); if (zombie) - hdr.len |= CPU_TO_BE32(GOSSIP_STORE_LEN_ZOMBIE_BIT); + hdr.flags |= CPU_TO_BE16(GOSSIP_STORE_ZOMBIE_BIT); hdr.crc = cpu_to_be32(crc32c(timestamp, msg, msglen)); hdr.timestamp = cpu_to_be32(timestamp); @@ -175,7 +176,7 @@ static u32 gossip_store_compact_offline(struct routing_state *rstate) size_t msglen; u8 *msg; - msglen = (be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_MASK); + msglen = be16_to_cpu(hdr.len); msg = tal_arr(NULL, u8, msglen); if (!read_all(old_fd, msg, msglen)) { status_broken("gossip_store_compact_offline: reading msg len %zu from store: %s", @@ -184,7 +185,7 @@ static u32 gossip_store_compact_offline(struct routing_state *rstate) goto close_and_delete; } - if (be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_DELETED_BIT) { + if (be16_to_cpu(hdr.flags) & GOSSIP_STORE_DELETED_BIT) { deleted++; tal_free(msg); continue; @@ -214,7 +215,7 @@ static u32 gossip_store_compact_offline(struct routing_state *rstate) /* Recalc msglen and header */ msglen = tal_bytelen(msg); - hdr.len = cpu_to_be32(msglen); + hdr.len = cpu_to_be16(msglen); hdr.crc = cpu_to_be32(crc32c(be32_to_cpu(hdr.timestamp), msg, msglen)); } @@ -317,7 +318,7 @@ static size_t transfer_store_msg(int from_fd, size_t from_off, int *type) { struct gossip_hdr hdr; - u32 msglen; + u16 flags, msglen; u8 *msg; const u8 *p; size_t tmplen; @@ -330,15 +331,14 @@ static size_t transfer_store_msg(int from_fd, size_t from_off, return 0; } - msglen = be32_to_cpu(hdr.len); - if (msglen & GOSSIP_STORE_LEN_DELETED_BIT) { + flags = be16_to_cpu(hdr.flags); + if (flags & GOSSIP_STORE_DELETED_BIT) { status_broken("Can't transfer deleted msg from gossip store @%zu", from_off); return 0; } - /* Ignore any non-length bits (e.g. push) */ - msglen &= GOSSIP_STORE_LEN_MASK; + msglen = be16_to_cpu(hdr.len); /* FIXME: Reuse buffer? */ msg = tal_arr(tmpctx, u8, sizeof(hdr) + msglen); @@ -454,11 +454,12 @@ bool gossip_store_compact(struct gossip_store *gs) /* Start by writing all channel announcements and updates. */ off = 1; while (pread(gs->fd, &hdr, sizeof(hdr), off) == sizeof(hdr)) { - u32 msglen, wlen; + u16 msglen; + u32 wlen; int msgtype; - msglen = (be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_MASK); - if (be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_DELETED_BIT) { + msglen = be16_to_cpu(hdr.len); + if (be16_to_cpu(hdr.flags) & GOSSIP_STORE_DELETED_BIT) { off += sizeof(hdr) + msglen; deleted++; continue; @@ -588,7 +589,10 @@ u64 gossip_store_add_private_update(struct gossip_store *gs, const u8 *update) /* Returns index of following entry. */ static u32 delete_by_index(struct gossip_store *gs, u32 index, int type) { - beint32_t belen; + struct { + beint16_t beflags; + beint16_t belen; + } hdr; /* Should never get here during loading! */ assert(gs->writable); @@ -601,21 +605,20 @@ static u32 delete_by_index(struct gossip_store *gs, u32 index, int type) assert(fromwire_peektype(msg) == type); #endif - if (pread(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) + if (pread(gs->fd, &hdr, sizeof(hdr), index) != sizeof(hdr)) status_failed(STATUS_FAIL_INTERNAL_ERROR, - "Failed reading len to delete @%u: %s", + "Failed reading flags & len to delete @%u: %s", index, strerror(errno)); - assert((be32_to_cpu(belen) & GOSSIP_STORE_LEN_DELETED_BIT) == 0); - belen |= cpu_to_be32(GOSSIP_STORE_LEN_DELETED_BIT); - if (pwrite(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) + assert((be16_to_cpu(hdr.beflags) & GOSSIP_STORE_DELETED_BIT) == 0); + hdr.beflags |= cpu_to_be16(GOSSIP_STORE_DELETED_BIT); + if (pwrite(gs->fd, &hdr.beflags, sizeof(hdr.beflags), index) != sizeof(hdr.beflags)) status_failed(STATUS_FAIL_INTERNAL_ERROR, - "Failed writing len to delete @%u: %s", + "Failed writing flags to delete @%u: %s", index, strerror(errno)); gs->deleted++; - return index + sizeof(struct gossip_hdr) - + (be32_to_cpu(belen) & GOSSIP_STORE_LEN_MASK); + return index + sizeof(struct gossip_hdr) + be16_to_cpu(hdr.belen); } void gossip_store_delete(struct gossip_store *gs, @@ -645,94 +648,59 @@ void gossip_store_mark_channel_deleted(struct gossip_store *gs, 0, false, false, false, NULL); } -/* Marks the length field of a channel_announcement with the zombie flag bit */ -void gossip_store_mark_channel_zombie(struct gossip_store *gs, - struct broadcastable *bcast) +static void mark_zombie(struct gossip_store *gs, + const struct broadcastable *bcast, + enum peer_wire expected_type) { - beint32_t belen; + beint16_t beflags; u32 index = bcast->index; + /* We assume flags is the first field! */ + BUILD_ASSERT(offsetof(struct gossip_hdr, flags) == 0); + /* Should never get here during loading! */ assert(gs->writable); - assert(index); #if DEVELOPER const u8 *msg = gossip_store_get(tmpctx, gs, index); - assert(fromwire_peektype(msg) == WIRE_CHANNEL_ANNOUNCEMENT); + assert(fromwire_peektype(msg) == expected_type); #endif - if (pread(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) + if (pread(gs->fd, &beflags, sizeof(beflags), index) != sizeof(beflags)) status_failed(STATUS_FAIL_INTERNAL_ERROR, - "Failed reading len to zombie channel @%u: %s", + "Failed reading flags to zombie %s @%u: %s", + peer_wire_name(expected_type), index, strerror(errno)); - assert((be32_to_cpu(belen) & GOSSIP_STORE_LEN_DELETED_BIT) == 0); - belen |= cpu_to_be32(GOSSIP_STORE_LEN_ZOMBIE_BIT); - if (pwrite(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) + assert((be16_to_cpu(beflags) & GOSSIP_STORE_DELETED_BIT) == 0); + beflags |= cpu_to_be16(GOSSIP_STORE_ZOMBIE_BIT); + if (pwrite(gs->fd, &beflags, sizeof(beflags), index) != sizeof(beflags)) status_failed(STATUS_FAIL_INTERNAL_ERROR, - "Failed writing len to zombie channel @%u: %s", + "Failed writing flags to zombie %s @%u: %s", + peer_wire_name(expected_type), index, strerror(errno)); } +/* Marks the length field of a channel_announcement with the zombie flag bit */ +void gossip_store_mark_channel_zombie(struct gossip_store *gs, + struct broadcastable *bcast) +{ + mark_zombie(gs, bcast, WIRE_CHANNEL_ANNOUNCEMENT); +} + /* Marks the length field of a channel_update with the zombie flag bit */ void gossip_store_mark_cupdate_zombie(struct gossip_store *gs, struct broadcastable *bcast) { - beint32_t belen; - u32 index = bcast->index; - - /* Should never get here during loading! */ - assert(gs->writable); - - assert(index); - -#if DEVELOPER - const u8 *msg = gossip_store_get(tmpctx, gs, index); - assert(fromwire_peektype(msg) == WIRE_CHANNEL_UPDATE); -#endif - - if (pread(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) - status_failed(STATUS_FAIL_INTERNAL_ERROR, - "Failed reading len to zombie channel update @%u: %s", - index, strerror(errno)); - - assert((be32_to_cpu(belen) & GOSSIP_STORE_LEN_DELETED_BIT) == 0); - belen |= cpu_to_be32(GOSSIP_STORE_LEN_ZOMBIE_BIT); - if (pwrite(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) - status_failed(STATUS_FAIL_INTERNAL_ERROR, - "Failed writing len to zombie channel update @%u: %s", - index, strerror(errno)); + mark_zombie(gs, bcast, WIRE_CHANNEL_UPDATE); } /* Marks the length field of a node_announcement with the zombie flag bit */ void gossip_store_mark_nannounce_zombie(struct gossip_store *gs, struct broadcastable *bcast) { - beint32_t belen; - u32 index = bcast->index; - - /* Should never get here during loading! */ - assert(gs->writable); - - assert(index); - -#if DEVELOPER - const u8 *msg = gossip_store_get(tmpctx, gs, index); - assert(fromwire_peektype(msg) == WIRE_NODE_ANNOUNCEMENT); -#endif - - if (pread(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) - status_failed(STATUS_FAIL_INTERNAL_ERROR, - "Failed reading len to zombie node announcement @%u: %s", - index, strerror(errno)); - - assert((be32_to_cpu(belen) & GOSSIP_STORE_LEN_DELETED_BIT) == 0); - belen |= cpu_to_be32(GOSSIP_STORE_LEN_ZOMBIE_BIT); - if (pwrite(gs->fd, &belen, sizeof(belen), index) != sizeof(belen)) - status_failed(STATUS_FAIL_INTERNAL_ERROR, - "Failed writing len to zombie channel update @%u: %s", - index, strerror(errno)); + mark_zombie(gs, bcast, WIRE_NODE_ANNOUNCEMENT); } const u8 *gossip_store_get(const tal_t *ctx, @@ -754,13 +722,13 @@ const u8 *gossip_store_get(const tal_t *ctx, offset, gs->len, strerror(errno)); } - if (be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_DELETED_BIT) + if (be16_to_cpu(hdr.flags) & GOSSIP_STORE_DELETED_BIT) status_failed(STATUS_FAIL_INTERNAL_ERROR, "gossip_store: get delete entry offset %"PRIu64 "/%"PRIu64"", offset, gs->len); - msglen = (be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_MASK); + msglen = be16_to_cpu(hdr.len); checksum = be32_to_cpu(hdr.crc); msg = tal_arr(ctx, u8, msglen); if (pread(gs->fd, msg, msglen, offset + sizeof(hdr)) != msglen) @@ -816,7 +784,9 @@ u32 gossip_store_load(struct routing_state *rstate, struct gossip_store *gs) gs->writable = false; while (pread(gs->fd, &hdr, sizeof(hdr), gs->len) == sizeof(hdr)) { - msglen = be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_MASK; + bool spam; + + msglen = be16_to_cpu(hdr.len); checksum = be32_to_cpu(hdr.crc); msg = tal_arr(tmpctx, u8, msglen); @@ -832,12 +802,13 @@ u32 gossip_store_load(struct routing_state *rstate, struct gossip_store *gs) } /* Skip deleted entries */ - if (be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_DELETED_BIT) { + if (be16_to_cpu(hdr.flags) & GOSSIP_STORE_DELETED_BIT) { /* Count includes deleted! */ gs->count++; gs->deleted++; goto next; } + spam = (be16_to_cpu(hdr.flags) & GOSSIP_STORE_RATELIMIT_BIT); switch (fromwire_peektype(msg)) { case WIRE_GOSSIP_STORE_PRIVATE_CHANNEL: { @@ -908,8 +879,7 @@ u32 gossip_store_load(struct routing_state *rstate, struct gossip_store *gs) case WIRE_CHANNEL_UPDATE: if (!routing_add_channel_update(rstate, take(msg), gs->len, - NULL, false, - be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_RATELIMIT_BIT)) { + NULL, false, spam)) { bad = "Bad channel_update"; goto badmsg; } @@ -918,8 +888,7 @@ u32 gossip_store_load(struct routing_state *rstate, struct gossip_store *gs) case WIRE_NODE_ANNOUNCEMENT: if (!routing_add_node_announcement(rstate, take(msg), gs->len, - NULL, NULL, - be32_to_cpu(hdr.len) & GOSSIP_STORE_LEN_RATELIMIT_BIT)) { + NULL, NULL, spam)) { bad = "Bad node_announcement"; goto badmsg; } diff --git a/plugins/test/run-route-overlong.c b/plugins/test/run-route-overlong.c index a5aaddfc995d..729fd5ac66ad 100644 --- a/plugins/test/run-route-overlong.c +++ b/plugins/test/run-route-overlong.c @@ -244,7 +244,8 @@ static void write_to_store(int store_fd, const u8 *msg) { struct gossip_hdr hdr; - hdr.len = cpu_to_be32(tal_count(msg)); + hdr.flags = cpu_to_be16(0); + hdr.len = cpu_to_be16(tal_count(msg)); /* We don't actually check these! */ hdr.crc = 0; hdr.timestamp = 0; From 22afc187c898e56925de38045eb16c7d7dcfd2f0 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:16 +1030 Subject: [PATCH 14/36] common/gossip_store: expose routine to read one header. This is useful when you're writing routines to scan it. Signed-off-by: Rusty Russell --- common/gossip_store.c | 47 ++++++++++++++++++++++++++++++++----------- common/gossip_store.h | 21 +++++++++++++++++++ 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/common/gossip_store.c b/common/gossip_store.c index cf7aaaae8521..361aaed38b6b 100644 --- a/common/gossip_store.c +++ b/common/gossip_store.c @@ -182,25 +182,48 @@ u8 *gossip_store_next(const tal_t *ctx, return msg; } -size_t find_gossip_store_end(int gossip_store_fd, size_t off) +/* We cheat and read first two bytes of message too. */ +struct hdr_and_type { + struct gossip_hdr hdr; + be16 type; +}; +/* Beware padding! */ +#define HDR_AND_TYPE_SIZE (sizeof(struct gossip_hdr) + sizeof(u16)) + +bool gossip_store_readhdr(int gossip_store_fd, size_t off, + size_t *len, + u32 *timestamp, + u16 *flags, + u16 *type) { - /* We cheat and read first two bytes of message too. */ - struct { - struct gossip_hdr hdr; - be16 type; - } buf; + struct hdr_and_type buf; int r; - while ((r = pread(gossip_store_fd, &buf, - sizeof(buf.hdr) + sizeof(buf.type), off)) - == sizeof(buf.hdr) + sizeof(buf.type)) { - u16 msglen = be16_to_cpu(buf.hdr.len); + r = pread(gossip_store_fd, &buf, HDR_AND_TYPE_SIZE, off); + if (r != HDR_AND_TYPE_SIZE) + return false; + *len = be16_to_cpu(buf.hdr.len); + if (flags) + *flags = be16_to_cpu(buf.hdr.flags); + if (timestamp) + *timestamp = be32_to_cpu(buf.hdr.timestamp); + if (type) + *type = be16_to_cpu(buf.type); + return true; +} + +size_t find_gossip_store_end(int gossip_store_fd, size_t off) +{ + size_t msglen; + u16 type; + while (gossip_store_readhdr(gossip_store_fd, off, + &msglen, NULL, NULL, &type)) { /* Don't swallow end marker! */ - if (buf.type == CPU_TO_BE16(WIRE_GOSSIP_STORE_ENDED)) + if (type == WIRE_GOSSIP_STORE_ENDED) break; - off += sizeof(buf.hdr) + msglen; + off += sizeof(struct gossip_hdr) + msglen; } return off; } diff --git a/common/gossip_store.h b/common/gossip_store.h index 6ef02e3c8641..2bc3340ae6e1 100644 --- a/common/gossip_store.h +++ b/common/gossip_store.h @@ -69,6 +69,27 @@ u8 *gossip_store_next(const tal_t *ctx, bool with_spam, size_t *off, size_t *end); +/** + * Direct store accessor: read gossip msg hdr from store. + * @gossip_store_fd: the readable file descriptor + * @off: the offset to read + * @len (out): the length of the message (not including header) + * @timestamp (out): if non-NULL, set to the timestamp. + * @flags (out): if non-NULL, set to the flags. + * @type (out): if non-NULL, set to the msg type. + * + * Returns false if there are no more gossip msgs. If you + * want to read the message, use gossip_store_next, if you + * want to skip, simply add sizeof(gossip_hdr) + *len to *off. + * Note: it's possible that entire record isn't there yet, + * so gossip_store_next can fail. + */ +bool gossip_store_readhdr(int gossip_store_fd, size_t off, + size_t *len, + u32 *timestamp, + u16 *flags, + u16 *type); + /** * Gossipd will be writing to this, and it's not atomic! Safest * way to find the "end" is to walk through. From 71c033bbb78c67dda4b0398e11788b7ee68491d2 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:16 +1030 Subject: [PATCH 15/36] common/gossip_store: move subdaemon-only routines to connectd. connectd is the only one who uses these routines now. The rest can be linked into a plugin. Signed-off-by: Rusty Russell --- common/gossip_store.c | 208 --------------------------------------- common/gossip_store.h | 21 ---- connectd/Makefile | 1 + connectd/gossip_store.c | 210 ++++++++++++++++++++++++++++++++++++++++ connectd/gossip_store.h | 27 ++++++ connectd/multiplex.c | 2 +- 6 files changed, 239 insertions(+), 230 deletions(-) create mode 100644 connectd/gossip_store.c create mode 100644 connectd/gossip_store.h diff --git a/common/gossip_store.c b/common/gossip_store.c index 361aaed38b6b..e209eedeaf02 100644 --- a/common/gossip_store.c +++ b/common/gossip_store.c @@ -10,178 +10,6 @@ #include #include -static bool timestamp_filter(u32 timestamp_min, u32 timestamp_max, - u32 timestamp) -{ - /* BOLT #7: - * - * - SHOULD send all gossip messages whose `timestamp` is greater or - * equal to `first_timestamp`, and less than `first_timestamp` plus - * `timestamp_range`. - */ - /* Note that we turn first_timestamp & timestamp_range into an inclusive range */ - return timestamp >= timestamp_min - && timestamp <= timestamp_max; -} - -static size_t reopen_gossip_store(int *gossip_store_fd, const u8 *msg) -{ - u64 equivalent_offset; - int newfd; - - if (!fromwire_gossip_store_ended(msg, &equivalent_offset)) - status_failed(STATUS_FAIL_GOSSIP_IO, - "Bad gossipd GOSSIP_STORE_ENDED msg: %s", - tal_hex(tmpctx, msg)); - - newfd = open(GOSSIP_STORE_FILENAME, O_RDONLY); - if (newfd < 0) - status_failed(STATUS_FAIL_INTERNAL_ERROR, - "Cannot open %s: %s", - GOSSIP_STORE_FILENAME, - strerror(errno)); - - status_debug("gossip_store at end, new fd moved to %"PRIu64, - equivalent_offset); - - close(*gossip_store_fd); - *gossip_store_fd = newfd; - return equivalent_offset; -} - -static bool public_msg_type(enum peer_wire type) -{ - /* This switch statement makes you think about new types as they - * are introduced. */ - switch (type) { - case WIRE_INIT: - case WIRE_ERROR: - case WIRE_WARNING: - case WIRE_PING: - case WIRE_PONG: - case WIRE_TX_ADD_INPUT: - case WIRE_TX_ADD_OUTPUT: - case WIRE_TX_REMOVE_INPUT: - case WIRE_TX_REMOVE_OUTPUT: - case WIRE_TX_COMPLETE: - case WIRE_TX_SIGNATURES: - case WIRE_OPEN_CHANNEL: - case WIRE_ACCEPT_CHANNEL: - case WIRE_FUNDING_CREATED: - case WIRE_FUNDING_SIGNED: - case WIRE_CHANNEL_READY: - case WIRE_OPEN_CHANNEL2: - case WIRE_ACCEPT_CHANNEL2: - case WIRE_INIT_RBF: - case WIRE_ACK_RBF: - case WIRE_SHUTDOWN: - case WIRE_CLOSING_SIGNED: - case WIRE_UPDATE_ADD_HTLC: - case WIRE_UPDATE_FULFILL_HTLC: - case WIRE_UPDATE_FAIL_HTLC: - case WIRE_UPDATE_FAIL_MALFORMED_HTLC: - case WIRE_COMMITMENT_SIGNED: - case WIRE_REVOKE_AND_ACK: - case WIRE_UPDATE_FEE: - case WIRE_UPDATE_BLOCKHEIGHT: - case WIRE_CHANNEL_REESTABLISH: - case WIRE_ANNOUNCEMENT_SIGNATURES: - case WIRE_QUERY_SHORT_CHANNEL_IDS: - case WIRE_REPLY_SHORT_CHANNEL_IDS_END: - case WIRE_QUERY_CHANNEL_RANGE: - case WIRE_REPLY_CHANNEL_RANGE: - case WIRE_GOSSIP_TIMESTAMP_FILTER: - case WIRE_ONION_MESSAGE: -#if EXPERIMENTAL_FEATURES - case WIRE_STFU: -#endif - return false; - case WIRE_CHANNEL_ANNOUNCEMENT: - case WIRE_NODE_ANNOUNCEMENT: - case WIRE_CHANNEL_UPDATE: - return true; - } - - /* Actually, we do have other (internal) messages. */ - return false; -} - -u8 *gossip_store_next(const tal_t *ctx, - int *gossip_store_fd, - u32 timestamp_min, u32 timestamp_max, - bool push_only, - bool with_spam, - size_t *off, size_t *end) -{ - u8 *msg = NULL; - size_t initial_off = *off; - - while (!msg) { - struct gossip_hdr hdr; - u16 msglen, flags; - u32 checksum, timestamp; - bool push, ratelimited; - int type, r; - - r = pread(*gossip_store_fd, &hdr, sizeof(hdr), *off); - if (r != sizeof(hdr)) - return NULL; - - msglen = be16_to_cpu(hdr.len); - flags = be16_to_cpu(hdr.flags); - push = (flags & GOSSIP_STORE_PUSH_BIT); - ratelimited = (flags & GOSSIP_STORE_RATELIMIT_BIT); - - /* Skip any deleted entries. */ - if (flags & GOSSIP_STORE_DELETED_BIT) { - *off += r + msglen; - continue; - } - - /* Skip any timestamp filtered */ - timestamp = be32_to_cpu(hdr.timestamp); - if (!push && - !timestamp_filter(timestamp_min, timestamp_max, - timestamp)) { - *off += r + msglen; - continue; - } - - checksum = be32_to_cpu(hdr.crc); - msg = tal_arr(ctx, u8, msglen); - r = pread(*gossip_store_fd, msg, msglen, *off + r); - if (r != msglen) - return tal_free(msg); - - if (checksum != crc32c(be32_to_cpu(hdr.timestamp), msg, msglen)) - status_failed(STATUS_FAIL_INTERNAL_ERROR, - "gossip_store: bad checksum at offset %zu" - "(was at %zu): %s", - *off, initial_off, tal_hex(tmpctx, msg)); - - /* Definitely processing it now */ - *off += sizeof(hdr) + msglen; - if (*off > *end) - *end = *off; - - type = fromwire_peektype(msg); - /* end can go backwards in this case! */ - if (type == WIRE_GOSSIP_STORE_ENDED) { - *off = *end = reopen_gossip_store(gossip_store_fd, msg); - msg = tal_free(msg); - /* Ignore gossipd internal messages. */ - } else if (!public_msg_type(type)) { - msg = tal_free(msg); - } else if (!push && push_only) { - msg = tal_free(msg); - } else if (!with_spam && ratelimited) { - msg = tal_free(msg); - } - } - - return msg; -} - /* We cheat and read first two bytes of message too. */ struct hdr_and_type { struct gossip_hdr hdr; @@ -227,39 +55,3 @@ size_t find_gossip_store_end(int gossip_store_fd, size_t off) } return off; } - -/* Keep seeking forward until we hit something >= timestamp */ -size_t find_gossip_store_by_timestamp(int gossip_store_fd, - size_t off, - u32 timestamp) -{ - /* We cheat and read first two bytes of message too. */ - struct { - struct gossip_hdr hdr; - be16 type; - } buf; - int r; - - while ((r = pread(gossip_store_fd, &buf, - sizeof(buf.hdr) + sizeof(buf.type), off)) - == sizeof(buf.hdr) + sizeof(buf.type)) { - u16 msglen = be16_to_cpu(buf.hdr.len); - u16 flags = be16_to_cpu(buf.hdr.flags); - u16 type = be16_to_cpu(buf.type); - - /* Don't swallow end marker! Reset, as they will call - * gossip_store_next and reopen file. */ - if (type == WIRE_GOSSIP_STORE_ENDED) - return 1; - - /* Only to-be-broadcast types have valid timestamps! */ - if (!(flags & GOSSIP_STORE_DELETED_BIT) - && public_msg_type(type) - && be32_to_cpu(buf.hdr.timestamp) >= timestamp) { - break; - } - - off += sizeof(buf.hdr) + msglen; - } - return off; -} diff --git a/common/gossip_store.h b/common/gossip_store.h index 2bc3340ae6e1..52bc9d98df56 100644 --- a/common/gossip_store.h +++ b/common/gossip_store.h @@ -55,20 +55,6 @@ struct gossip_hdr { beint32_t timestamp; /* timestamp of msg. */ }; -/** - * Direct store accessor: loads gossip msg from store. - * - * Returns NULL if there are no more gossip msgs. - * Updates *end if the known end of file has moved. - * Updates *gossip_store_fd if file has been compacted. - */ -u8 *gossip_store_next(const tal_t *ctx, - int *gossip_store_fd, - u32 timestamp_min, u32 timestamp_max, - bool push_only, - bool with_spam, - size_t *off, size_t *end); - /** * Direct store accessor: read gossip msg hdr from store. * @gossip_store_fd: the readable file descriptor @@ -96,11 +82,4 @@ bool gossip_store_readhdr(int gossip_store_fd, size_t off, * @old_end: 1 if no previous end. */ size_t find_gossip_store_end(int gossip_store_fd, size_t old_end); - -/** - * Return offset of first entry >= this timestamp. - */ -size_t find_gossip_store_by_timestamp(int gossip_store_fd, - size_t off, - u32 timestamp); #endif /* LIGHTNING_COMMON_GOSSIP_STORE_H */ diff --git a/connectd/Makefile b/connectd/Makefile index b8c9b3d341ac..a0b72ee8daf2 100644 --- a/connectd/Makefile +++ b/connectd/Makefile @@ -5,6 +5,7 @@ CONNECTD_HEADERS := connectd/connectd_wiregen.h \ connectd/connectd.h \ connectd/peer_exchange_initmsg.h \ connectd/handshake.h \ + connectd/gossip_store.h \ connectd/gossip_rcvd_filter.h \ connectd/multiplex.h \ connectd/netaddress.h \ diff --git a/connectd/gossip_store.c b/connectd/gossip_store.c new file mode 100644 index 000000000000..02af672eded1 --- /dev/null +++ b/connectd/gossip_store.c @@ -0,0 +1,210 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool timestamp_filter(u32 timestamp_min, u32 timestamp_max, + u32 timestamp) +{ + /* BOLT #7: + * + * - SHOULD send all gossip messages whose `timestamp` is greater or + * equal to `first_timestamp`, and less than `first_timestamp` plus + * `timestamp_range`. + */ + /* Note that we turn first_timestamp & timestamp_range into an inclusive range */ + return timestamp >= timestamp_min + && timestamp <= timestamp_max; +} + +static size_t reopen_gossip_store(int *gossip_store_fd, const u8 *msg) +{ + u64 equivalent_offset; + int newfd; + + if (!fromwire_gossip_store_ended(msg, &equivalent_offset)) + status_failed(STATUS_FAIL_GOSSIP_IO, + "Bad gossipd GOSSIP_STORE_ENDED msg: %s", + tal_hex(tmpctx, msg)); + + newfd = open(GOSSIP_STORE_FILENAME, O_RDONLY); + if (newfd < 0) + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "Cannot open %s: %s", + GOSSIP_STORE_FILENAME, + strerror(errno)); + + status_debug("gossip_store at end, new fd moved to %"PRIu64, + equivalent_offset); + + close(*gossip_store_fd); + *gossip_store_fd = newfd; + return equivalent_offset; +} + +static bool public_msg_type(enum peer_wire type) +{ + /* This switch statement makes you think about new types as they + * are introduced. */ + switch (type) { + case WIRE_INIT: + case WIRE_ERROR: + case WIRE_WARNING: + case WIRE_PING: + case WIRE_PONG: + case WIRE_TX_ADD_INPUT: + case WIRE_TX_ADD_OUTPUT: + case WIRE_TX_REMOVE_INPUT: + case WIRE_TX_REMOVE_OUTPUT: + case WIRE_TX_COMPLETE: + case WIRE_TX_SIGNATURES: + case WIRE_OPEN_CHANNEL: + case WIRE_ACCEPT_CHANNEL: + case WIRE_FUNDING_CREATED: + case WIRE_FUNDING_SIGNED: + case WIRE_CHANNEL_READY: + case WIRE_OPEN_CHANNEL2: + case WIRE_ACCEPT_CHANNEL2: + case WIRE_INIT_RBF: + case WIRE_ACK_RBF: + case WIRE_SHUTDOWN: + case WIRE_CLOSING_SIGNED: + case WIRE_UPDATE_ADD_HTLC: + case WIRE_UPDATE_FULFILL_HTLC: + case WIRE_UPDATE_FAIL_HTLC: + case WIRE_UPDATE_FAIL_MALFORMED_HTLC: + case WIRE_COMMITMENT_SIGNED: + case WIRE_REVOKE_AND_ACK: + case WIRE_UPDATE_FEE: + case WIRE_UPDATE_BLOCKHEIGHT: + case WIRE_CHANNEL_REESTABLISH: + case WIRE_ANNOUNCEMENT_SIGNATURES: + case WIRE_QUERY_SHORT_CHANNEL_IDS: + case WIRE_REPLY_SHORT_CHANNEL_IDS_END: + case WIRE_QUERY_CHANNEL_RANGE: + case WIRE_REPLY_CHANNEL_RANGE: + case WIRE_GOSSIP_TIMESTAMP_FILTER: + case WIRE_ONION_MESSAGE: +#if EXPERIMENTAL_FEATURES + case WIRE_STFU: +#endif + return false; + case WIRE_CHANNEL_ANNOUNCEMENT: + case WIRE_NODE_ANNOUNCEMENT: + case WIRE_CHANNEL_UPDATE: + return true; + } + + /* Actually, we do have other (internal) messages. */ + return false; +} + +u8 *gossip_store_next(const tal_t *ctx, + int *gossip_store_fd, + u32 timestamp_min, u32 timestamp_max, + bool push_only, + bool with_spam, + size_t *off, size_t *end) +{ + u8 *msg = NULL; + size_t initial_off = *off; + + while (!msg) { + struct gossip_hdr hdr; + u16 msglen, flags; + u32 checksum, timestamp; + bool push, ratelimited; + int type, r; + + r = pread(*gossip_store_fd, &hdr, sizeof(hdr), *off); + if (r != sizeof(hdr)) + return NULL; + + msglen = be16_to_cpu(hdr.len); + flags = be16_to_cpu(hdr.flags); + push = (flags & GOSSIP_STORE_PUSH_BIT); + ratelimited = (flags & GOSSIP_STORE_RATELIMIT_BIT); + + /* Skip any deleted entries. */ + if (flags & GOSSIP_STORE_DELETED_BIT) { + *off += r + msglen; + continue; + } + + /* Skip any timestamp filtered */ + timestamp = be32_to_cpu(hdr.timestamp); + if (!push && + !timestamp_filter(timestamp_min, timestamp_max, + timestamp)) { + *off += r + msglen; + continue; + } + + checksum = be32_to_cpu(hdr.crc); + msg = tal_arr(ctx, u8, msglen); + r = pread(*gossip_store_fd, msg, msglen, *off + r); + if (r != msglen) + return tal_free(msg); + + if (checksum != crc32c(be32_to_cpu(hdr.timestamp), msg, msglen)) + status_failed(STATUS_FAIL_INTERNAL_ERROR, + "gossip_store: bad checksum at offset %zu" + "(was at %zu): %s", + *off, initial_off, tal_hex(tmpctx, msg)); + + /* Definitely processing it now */ + *off += sizeof(hdr) + msglen; + if (*off > *end) + *end = *off; + + type = fromwire_peektype(msg); + /* end can go backwards in this case! */ + if (type == WIRE_GOSSIP_STORE_ENDED) { + *off = *end = reopen_gossip_store(gossip_store_fd, msg); + msg = tal_free(msg); + /* Ignore gossipd internal messages. */ + } else if (!public_msg_type(type)) { + msg = tal_free(msg); + } else if (!push && push_only) { + msg = tal_free(msg); + } else if (!with_spam && ratelimited) { + msg = tal_free(msg); + } + } + + return msg; +} + +/* Keep seeking forward until we hit something >= timestamp */ +size_t find_gossip_store_by_timestamp(int gossip_store_fd, + size_t off, + u32 timestamp) +{ + u16 type, flags; + u32 ts; + size_t msglen; + + while (gossip_store_readhdr(gossip_store_fd, off, + &msglen, &ts, &flags, &type)) { + /* Don't swallow end marker! Reset, as they will call + * gossip_store_next and reopen file. */ + if (type == WIRE_GOSSIP_STORE_ENDED) + return 1; + + /* Only to-be-broadcast types have valid timestamps! */ + if (!(flags & GOSSIP_STORE_DELETED_BIT) + && public_msg_type(type) + && ts >= timestamp) { + break; + } + + off += sizeof(struct gossip_hdr) + msglen; + } + return off; +} diff --git a/connectd/gossip_store.h b/connectd/gossip_store.h new file mode 100644 index 000000000000..21a0eb7549d2 --- /dev/null +++ b/connectd/gossip_store.h @@ -0,0 +1,27 @@ +#ifndef LIGHTNING_CONNECTD_GOSSIP_STORE_H +#define LIGHTNING_CONNECTD_GOSSIP_STORE_H +#include "config.h" +#include + +/** + * Direct store accessor: loads gossip msg from store. + * + * Returns NULL if there are no more gossip msgs. + * Updates *end if the known end of file has moved. + * Updates *gossip_store_fd if file has been compacted. + */ +u8 *gossip_store_next(const tal_t *ctx, + int *gossip_store_fd, + u32 timestamp_min, u32 timestamp_max, + bool push_only, + bool with_spam, + size_t *off, size_t *end); + +/** + * Return offset of first entry >= this timestamp. + */ +size_t find_gossip_store_by_timestamp(int gossip_store_fd, + size_t off, + u32 timestamp); + +#endif /* LIGHTNING_CONNECTD_GOSSIP_STORE_H */ diff --git a/connectd/multiplex.c b/connectd/multiplex.c index ecfcffe538f5..90fc34a2439c 100644 --- a/connectd/multiplex.c +++ b/connectd/multiplex.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -23,6 +22,7 @@ #include #include #include +#include #include #include #include From abb0564770abc448e73fcfe4f42f0abf8ced1ee2 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:16 +1030 Subject: [PATCH 16/36] tools/fromschema.py: don't try to handle more complex cases. We only handle top-level objects with an array of objects: make sure it is one before we call the routines. Signed-off-by: Rusty Russell --- doc/lightning-addgossip.7.md | 2 +- doc/lightning-autoclean-once.7.md | 2 +- doc/lightning-autoclean-status.7.md | 2 +- doc/lightning-batching.7.md | 2 +- doc/lightning-bkpr-channelsapy.7.md | 2 +- doc/lightning-bkpr-dumpincomecsv.7.md | 2 +- doc/lightning-bkpr-inspect.7.md | 2 +- doc/lightning-bkpr-listaccountevents.7.md | 2 +- doc/lightning-bkpr-listbalances.7.md | 2 +- doc/lightning-bkpr-listincome.7.md | 2 +- doc/lightning-check.7.md | 2 +- doc/lightning-checkmessage.7.md | 2 +- doc/lightning-close.7.md | 2 +- doc/lightning-commando-rune.7.md | 2 +- doc/lightning-connect.7.md | 2 +- doc/lightning-createinvoice.7.md | 2 +- doc/lightning-createonion.7.md | 2 +- doc/lightning-datastore.7.md | 2 +- doc/lightning-decode.7.md | 2 +- doc/lightning-decodepay.7.md | 2 +- doc/lightning-deldatastore.7.md | 2 +- doc/lightning-delexpiredinvoice.7.md | 2 +- doc/lightning-delforward.7.md | 2 +- doc/lightning-delinvoice.7.md | 2 +- doc/lightning-delpay.7.md | 2 +- doc/lightning-disableoffer.7.md | 2 +- doc/lightning-disconnect.7.md | 2 +- doc/lightning-feerates.7.md | 2 +- doc/lightning-fetchinvoice.7.md | 2 +- doc/lightning-fundchannel.7.md | 2 +- doc/lightning-fundchannel_cancel.7.md | 2 +- doc/lightning-fundchannel_complete.7.md | 2 +- doc/lightning-fundchannel_start.7.md | 2 +- doc/lightning-funderupdate.7.md | 2 +- doc/lightning-fundpsbt.7.md | 2 +- doc/lightning-getinfo.7.md | 2 +- doc/lightning-getlog.7.md | 2 +- doc/lightning-getroute.7.md | 2 +- doc/lightning-help.7.md | 2 +- doc/lightning-invoice.7.md | 2 +- doc/lightning-keysend.7.md | 2 +- doc/lightning-listchannels.7.md | 2 +- doc/lightning-listconfigs.7.md | 2 +- doc/lightning-listdatastore.7.md | 2 +- doc/lightning-listforwards.7.md | 2 +- doc/lightning-listfunds.7.md | 2 +- doc/lightning-listhtlcs.7.md | 2 +- doc/lightning-listinvoices.7.md | 2 +- doc/lightning-listnodes.7.md | 2 +- doc/lightning-listoffers.7.md | 2 +- doc/lightning-listpays.7.md | 2 +- doc/lightning-listpeerchannels.7.md | 2 +- doc/lightning-listpeers.7.md | 2 +- doc/lightning-listsendpays.7.md | 2 +- doc/lightning-listtransactions.7.md | 2 +- doc/lightning-makesecret.7.md | 2 +- doc/lightning-multifundchannel.7.md | 2 +- doc/lightning-multiwithdraw.7.md | 2 +- doc/lightning-newaddr.7.md | 2 +- doc/lightning-notifications.7.md | 2 +- doc/lightning-offer.7.md | 2 +- doc/lightning-openchannel_abort.7.md | 2 +- doc/lightning-openchannel_bump.7.md | 2 +- doc/lightning-openchannel_init.7.md | 2 +- doc/lightning-openchannel_signed.7.md | 2 +- doc/lightning-openchannel_update.7.md | 2 +- doc/lightning-parsefeerate.7.md | 2 +- doc/lightning-pay.7.md | 2 +- doc/lightning-ping.7.md | 2 +- doc/lightning-plugin.7.md | 2 +- doc/lightning-preapproveinvoice.7.md | 2 +- doc/lightning-preapprovekeysend.7.md | 2 +- doc/lightning-reserveinputs.7.md | 2 +- doc/lightning-sendcustommsg.7.md | 2 +- doc/lightning-sendinvoice.7.md | 2 +- doc/lightning-sendonion.7.md | 2 +- doc/lightning-sendonionmessage.7.md | 2 +- doc/lightning-sendpay.7.md | 2 +- doc/lightning-sendpsbt.7.md | 2 +- doc/lightning-setchannel.7.md | 2 +- doc/lightning-signmessage.7.md | 2 +- doc/lightning-signpsbt.7.md | 2 +- doc/lightning-stop.7.md | 2 +- doc/lightning-txdiscard.7.md | 2 +- doc/lightning-txprepare.7.md | 2 +- doc/lightning-txsend.7.md | 2 +- doc/lightning-unreserveinputs.7.md | 2 +- doc/lightning-utxopsbt.7.md | 2 +- doc/lightning-waitanyinvoice.7.md | 2 +- doc/lightning-waitblockheight.7.md | 2 +- doc/lightning-waitinvoice.7.md | 2 +- doc/lightning-waitsendpay.7.md | 2 +- doc/lightning-withdraw.7.md | 2 +- tools/fromschema.py | 2 +- 94 files changed, 94 insertions(+), 94 deletions(-) diff --git a/doc/lightning-addgossip.7.md b/doc/lightning-addgossip.7.md index af2d44a1ef8e..e111b189236a 100644 --- a/doc/lightning-addgossip.7.md +++ b/doc/lightning-addgossip.7.md @@ -42,4 +42,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ec98523e094209b75eeeb620d8f2a64409dafe6ba21baf3a89ade514b285d202) +[comment]: # ( SHA256STAMP:41d0ca6a956520453538c8ad5c5afce681540f4ce26017570cdc2356c3aab599) diff --git a/doc/lightning-autoclean-once.7.md b/doc/lightning-autoclean-once.7.md index 825a63c0c572..b3e1e8445ad7 100644 --- a/doc/lightning-autoclean-once.7.md +++ b/doc/lightning-autoclean-once.7.md @@ -67,4 +67,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:98305a03bfeabffa053acbec0ed7d85ad9c0e342aad2b90aec260a51f0842e42) +[comment]: # ( SHA256STAMP:1f2819ff0d1a246efbe6dbd027083cb9c6dc0eedb4c7e44ead5d399c5fda07d4) diff --git a/doc/lightning-autoclean-status.7.md b/doc/lightning-autoclean-status.7.md index 4b5d4797ba34..7a04fdde2737 100644 --- a/doc/lightning-autoclean-status.7.md +++ b/doc/lightning-autoclean-status.7.md @@ -91,4 +91,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:32dcce7526a1ebb45e7018df70434cdbe2fe3a78ac64c1d257a85e49d7cfa755) +[comment]: # ( SHA256STAMP:0d997b9f41940245f25d5eb8ce9b9678c305bbdd3941cd17a1b2ba4c7d42310b) diff --git a/doc/lightning-batching.7.md b/doc/lightning-batching.7.md index d8a25f8077bc..2cba44e345b9 100644 --- a/doc/lightning-batching.7.md +++ b/doc/lightning-batching.7.md @@ -53,4 +53,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ec98523e094209b75eeeb620d8f2a64409dafe6ba21baf3a89ade514b285d202) +[comment]: # ( SHA256STAMP:41d0ca6a956520453538c8ad5c5afce681540f4ce26017570cdc2356c3aab599) diff --git a/doc/lightning-bkpr-channelsapy.7.md b/doc/lightning-bkpr-channelsapy.7.md index ceb8aaba91f3..cb1dd9bf9855 100644 --- a/doc/lightning-bkpr-channelsapy.7.md +++ b/doc/lightning-bkpr-channelsapy.7.md @@ -65,4 +65,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:c4521fe6da034f419cc94e1e0e969e7c230e8342412ea91c929ba30812175451) +[comment]: # ( SHA256STAMP:25085a4f855ea54ca1389d1776f48bd285d4cd7f8267f782fe1eb99c5d416d60) diff --git a/doc/lightning-bkpr-dumpincomecsv.7.md b/doc/lightning-bkpr-dumpincomecsv.7.md index d8388f6c578c..1e34a516650d 100644 --- a/doc/lightning-bkpr-dumpincomecsv.7.md +++ b/doc/lightning-bkpr-dumpincomecsv.7.md @@ -57,4 +57,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:23c9bdc615f0fc8475b16a5a79607b8da528c9a218c32c8f2ccab835fe0b9078) +[comment]: # ( SHA256STAMP:ae67965eda25bd27a56f8284d591e9c3760f859872b9805caf22485ee7b8c4e8) diff --git a/doc/lightning-bkpr-inspect.7.md b/doc/lightning-bkpr-inspect.7.md index 37d724ffa262..5dfbe02e46f3 100644 --- a/doc/lightning-bkpr-inspect.7.md +++ b/doc/lightning-bkpr-inspect.7.md @@ -52,4 +52,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:6b3c960fb6d159ba5df5bd85960a8145e6dec7487ac84837127f9d461c1e8103) +[comment]: # ( SHA256STAMP:431ddf80b4cc4ad778a0b7e720dc282671a34f62de9ceedb0f411277bdda91be) diff --git a/doc/lightning-bkpr-listaccountevents.7.md b/doc/lightning-bkpr-listaccountevents.7.md index 700e01856e62..4c7b8850bc89 100644 --- a/doc/lightning-bkpr-listaccountevents.7.md +++ b/doc/lightning-bkpr-listaccountevents.7.md @@ -71,4 +71,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:b075f8f54879e8b2f1d9d1b93f08f0fb7a2be6fc7c5bff051b3c1d575596da81) +[comment]: # ( SHA256STAMP:e7c828540de32dbcc9c3b5f17c0f559a1217d34834d3fe8f3b8f79a9aacea9f5) diff --git a/doc/lightning-bkpr-listbalances.7.md b/doc/lightning-bkpr-listbalances.7.md index a68912c03c8d..efac73eb456f 100644 --- a/doc/lightning-bkpr-listbalances.7.md +++ b/doc/lightning-bkpr-listbalances.7.md @@ -53,4 +53,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:d274d8cde5ec727630d6529664f0a1e287968e7071d3e2d7ca041bd5ee681d73) +[comment]: # ( SHA256STAMP:e3a929a7568cceeb54d94807dd41ca057b0f3c7320a98d8b3a405b6bd60c77df) diff --git a/doc/lightning-bkpr-listincome.7.md b/doc/lightning-bkpr-listincome.7.md index 337bd83a9a55..4317619a51b6 100644 --- a/doc/lightning-bkpr-listincome.7.md +++ b/doc/lightning-bkpr-listincome.7.md @@ -57,4 +57,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:3af2f2f05de9f698154d9fb90d7722bc7b47b8c220af58e90034636219198c87) +[comment]: # ( SHA256STAMP:ec2d5cc8d55017dcad2f9bfc41e287debe710c50134e581a1cb8af2986c41dcc) diff --git a/doc/lightning-check.7.md b/doc/lightning-check.7.md index 0fd7b9e20525..8b62e0aa26da 100644 --- a/doc/lightning-check.7.md +++ b/doc/lightning-check.7.md @@ -41,4 +41,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:632a4a2cdaac10cdde1719f70bea672f3dc7c292e25395c6235255b1e2a11b28) +[comment]: # ( SHA256STAMP:069205c0316e7096044ef71b3fa2525e389c907b45c73177469d06e69d03873c) diff --git a/doc/lightning-checkmessage.7.md b/doc/lightning-checkmessage.7.md index 7e82bd36188e..fac1076fe997 100644 --- a/doc/lightning-checkmessage.7.md +++ b/doc/lightning-checkmessage.7.md @@ -50,4 +50,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ad6f3db0c6da357d6849aae546d50439e30cc43329150f02c1801a74e8c1c338) +[comment]: # ( SHA256STAMP:4a7c148e1b7f321a7710f540de2d8418850f1a6269badab8cbe47545c41f4d01) diff --git a/doc/lightning-close.7.md b/doc/lightning-close.7.md index 07a2ab3ef21e..8107124fbbd5 100644 --- a/doc/lightning-close.7.md +++ b/doc/lightning-close.7.md @@ -135,4 +135,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:d226164144e972628ff14d72e80a86b0016902851fde8f696f869abce697ff82) +[comment]: # ( SHA256STAMP:87455886c43ec55b784eb21370098d9b242856bf6756c1828abbcd24cad87bda) diff --git a/doc/lightning-commando-rune.7.md b/doc/lightning-commando-rune.7.md index b38e01338a98..1f61f79b5fca 100644 --- a/doc/lightning-commando-rune.7.md +++ b/doc/lightning-commando-rune.7.md @@ -218,4 +218,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:edf7087118018f9cc94b37aa3bb5bf976364d095d3d6bd1472cea171d9605183) +[comment]: # ( SHA256STAMP:7064d2dcc37af3fe83739a11da57400b5c1faef51095b8dacfba6a4312fc9d25) diff --git a/doc/lightning-connect.7.md b/doc/lightning-connect.7.md index d9837469871f..fc7b5005f196 100644 --- a/doc/lightning-connect.7.md +++ b/doc/lightning-connect.7.md @@ -104,4 +104,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:c1361f5a2b1cff63b7ca5367c2d8b04c30f617c8a5943160afff620d5a099faa) +[comment]: # ( SHA256STAMP:25d387fccf09c23ffa9185e8eb6d37b676ca9bc31761eabe7b16e6e1dbeec4c1) diff --git a/doc/lightning-createinvoice.7.md b/doc/lightning-createinvoice.7.md index 5a3569cd0e03..2652d8dc7503 100644 --- a/doc/lightning-createinvoice.7.md +++ b/doc/lightning-createinvoice.7.md @@ -75,4 +75,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:e770524d66bd9c0eed2ebba2a9b73aab9cc6be10e2e1500f13506624b1703bf1) +[comment]: # ( SHA256STAMP:1da48df402e8ea4fa1e5901d3272dc9d09f32fd62b03cf90393778add281b732) diff --git a/doc/lightning-createonion.7.md b/doc/lightning-createonion.7.md index 1400b6d9c1d9..b240257f1e64 100644 --- a/doc/lightning-createonion.7.md +++ b/doc/lightning-createonion.7.md @@ -132,4 +132,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:9cb9d5ecae6a2480ba26ae9a6b3d894c81c4f7dbdcfbcc2132e7df5958886616) +[comment]: # ( SHA256STAMP:31882493866e2d3fea36ab68e610637609967fc1095bc926da9bdfbd578de08b) diff --git a/doc/lightning-datastore.7.md b/doc/lightning-datastore.7.md index ee2b4963943e..9f22e75675c5 100644 --- a/doc/lightning-datastore.7.md +++ b/doc/lightning-datastore.7.md @@ -66,4 +66,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:6f79d42d7c47fb7fde8debdadde78a658c3502299dfbbf7422cf19fc482f4019) +[comment]: # ( SHA256STAMP:36976b15cda1e013713b88f2411da5900b169d99c87ee30b5e8389e45a0df68a) diff --git a/doc/lightning-decode.7.md b/doc/lightning-decode.7.md index c4a17f0a17ca..2c4bfc37e122 100644 --- a/doc/lightning-decode.7.md +++ b/doc/lightning-decode.7.md @@ -303,4 +303,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:a8843027b18a1d54efcf0ca423fc6fbe0c2136d32fa3d8c552a875b6844dd950) +[comment]: # ( SHA256STAMP:25247da6ea63f7fa229b7d25afd8cc2558ded16219c4a49ca3567dbb522f0153) diff --git a/doc/lightning-decodepay.7.md b/doc/lightning-decodepay.7.md index 1f1dc84f0849..cb256b184753 100644 --- a/doc/lightning-decodepay.7.md +++ b/doc/lightning-decodepay.7.md @@ -71,4 +71,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:2b48a4c96cbb86a667fb920bbcf477b2971795eaf394f72b436351d6063441f2) +[comment]: # ( SHA256STAMP:1d4a7f4577ffa26f34da9ea3a0ed2f7da26f86a4ce554b8bf20785a3de20fdfe) diff --git a/doc/lightning-deldatastore.7.md b/doc/lightning-deldatastore.7.md index 20b32bfa56c2..e4bf20bd8dad 100644 --- a/doc/lightning-deldatastore.7.md +++ b/doc/lightning-deldatastore.7.md @@ -49,4 +49,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:67036bda933bda35532ea55be1b6140785132d8d747094aba09d47fed40ec5ea) +[comment]: # ( SHA256STAMP:62e5e173120950d1e059e4b53510ed0d1f103d5edb52f9e378e23625e7791ac5) diff --git a/doc/lightning-delexpiredinvoice.7.md b/doc/lightning-delexpiredinvoice.7.md index a1575a556ba5..1c67704edd80 100644 --- a/doc/lightning-delexpiredinvoice.7.md +++ b/doc/lightning-delexpiredinvoice.7.md @@ -38,4 +38,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:6c100f08fe4cb4aa4f1b78243bfeb4414db2ff9bb5145fdb9eca7c9a4037d0e3) +[comment]: # ( SHA256STAMP:bea6cd45f3e8c912180dd76a69a13a2f1d2ba88adab001cfa161700cfc8288b4) diff --git a/doc/lightning-delforward.7.md b/doc/lightning-delforward.7.md index ef6391a7b25a..168b80caa3c0 100644 --- a/doc/lightning-delforward.7.md +++ b/doc/lightning-delforward.7.md @@ -55,4 +55,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:a2b84b83e10b81fd82a2bed20874f707de6002c076a4276d4f6ff30772ad3e88) +[comment]: # ( SHA256STAMP:636acc798ed7ae1cd307ada4dbde424c1ed8aa514600bec9adeacd5778f4d036) diff --git a/doc/lightning-delinvoice.7.md b/doc/lightning-delinvoice.7.md index 45af485dc2bd..134c9a2a6c90 100644 --- a/doc/lightning-delinvoice.7.md +++ b/doc/lightning-delinvoice.7.md @@ -81,4 +81,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:8748474d2323baf828d343de470f0f890e92d59240cd0f56bc6f0a3a802b7c70) +[comment]: # ( SHA256STAMP:453b05959f4f08b5d7f09d2ba6f5504d8775d3e8a81b7e02b20f755dfe3ded88) diff --git a/doc/lightning-delpay.7.md b/doc/lightning-delpay.7.md index a35746514143..1bc7d3ef954b 100644 --- a/doc/lightning-delpay.7.md +++ b/doc/lightning-delpay.7.md @@ -107,4 +107,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:996cdabe39a1c684f92ffee47334da2ef921ecb1bb411b063c8a748e172c3f20) +[comment]: # ( SHA256STAMP:c6d248396e04a0ef3506dbd5319231cf8d4cb2522a4e039d5e3622aed5ab4496) diff --git a/doc/lightning-disableoffer.7.md b/doc/lightning-disableoffer.7.md index f9bb46085524..92ccddcc01a6 100644 --- a/doc/lightning-disableoffer.7.md +++ b/doc/lightning-disableoffer.7.md @@ -74,4 +74,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:7a813b5ae0c4ae2a9a77063d710ee7d4ef621d828e6a4a57c59d62d3f2091c89) +[comment]: # ( SHA256STAMP:339f2e5a5a587414ba6ad3e312c6cff5525bae86addb76bb5656665eeeef3dd6) diff --git a/doc/lightning-disconnect.7.md b/doc/lightning-disconnect.7.md index 21576c1f720f..2c4df804a1ab 100644 --- a/doc/lightning-disconnect.7.md +++ b/doc/lightning-disconnect.7.md @@ -59,4 +59,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ec98523e094209b75eeeb620d8f2a64409dafe6ba21baf3a89ade514b285d202) +[comment]: # ( SHA256STAMP:41d0ca6a956520453538c8ad5c5afce681540f4ce26017570cdc2356c3aab599) diff --git a/doc/lightning-feerates.7.md b/doc/lightning-feerates.7.md index edcfb9f422a8..db4139d929bd 100644 --- a/doc/lightning-feerates.7.md +++ b/doc/lightning-feerates.7.md @@ -121,4 +121,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:4b1fa6f6ac4a6f9218486d9b42c8bc684fd48ad615b5644a0cd6650719d32a78) +[comment]: # ( SHA256STAMP:a34af89895413ee57defa1df715d9e50d34a49196972a81b31a4666b4fc05a10) diff --git a/doc/lightning-fetchinvoice.7.md b/doc/lightning-fetchinvoice.7.md index 99f4e0366d21..f09ec45f347f 100644 --- a/doc/lightning-fetchinvoice.7.md +++ b/doc/lightning-fetchinvoice.7.md @@ -89,4 +89,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:8946b80c01151cfbb8c363a6c95e57e17cae39ad0fc9fa79646cd74af3548a5d) +[comment]: # ( SHA256STAMP:c14601b72fda248cbc8b86253bda6399a0886fc1fd0332d3acbc1d8800342126) diff --git a/doc/lightning-fundchannel.7.md b/doc/lightning-fundchannel.7.md index d0b68b1371ca..5ee6c46a2031 100644 --- a/doc/lightning-fundchannel.7.md +++ b/doc/lightning-fundchannel.7.md @@ -121,4 +121,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:4c393a5fb6bd920dfbe2bbf79120f5d11bc9f5b18d71a70a519d245b0c366ee6) +[comment]: # ( SHA256STAMP:a8329cdb3f13f5bd0047824bed82c2e10516af2735dc59aa2cd71e4cc4f0250a) diff --git a/doc/lightning-fundchannel_cancel.7.md b/doc/lightning-fundchannel_cancel.7.md index 30eebcbe457b..9f027fa41dc6 100644 --- a/doc/lightning-fundchannel_cancel.7.md +++ b/doc/lightning-fundchannel_cancel.7.md @@ -60,4 +60,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ac584f54804ac2b3d28ab4ddb9cd6ebb9deb36d49a431e25b1d63cfb3a0480d3) +[comment]: # ( SHA256STAMP:9e0f448bb3c97434118d87044fc04ae4b573ff14877ab96eddceecee21c1c4f4) diff --git a/doc/lightning-fundchannel_complete.7.md b/doc/lightning-fundchannel_complete.7.md index c6c52815b5aa..6de550cd1cfc 100644 --- a/doc/lightning-fundchannel_complete.7.md +++ b/doc/lightning-fundchannel_complete.7.md @@ -62,4 +62,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:c7c616115c90f39b896ac5cfba7d7bbdb9ec9e762e3de9527e8222d1a3a78e44) +[comment]: # ( SHA256STAMP:d264eb570b8e743cf6fbd29d78586224cf565c013d1f7682a01db53858b13467) diff --git a/doc/lightning-fundchannel_start.7.md b/doc/lightning-fundchannel_start.7.md index a66ea4c085a5..3a00230ecdb1 100644 --- a/doc/lightning-fundchannel_start.7.md +++ b/doc/lightning-fundchannel_start.7.md @@ -85,4 +85,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:f19bf6fedcc7c62a4422ef06abe1eb5b5d10aa4345b6a39ae5fce7408448ae0a) +[comment]: # ( SHA256STAMP:ed685f91a9242a38a2d48b82ed7ba063a1a4d754d95283ad232cbe7d12471659) diff --git a/doc/lightning-funderupdate.7.md b/doc/lightning-funderupdate.7.md index c466119c3998..d7da56c74677 100644 --- a/doc/lightning-funderupdate.7.md +++ b/doc/lightning-funderupdate.7.md @@ -147,4 +147,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:3ff71c0a62f350d099dbe5b3e5c4e20b486214a23333fcf5bcd6e7b062175ff9) +[comment]: # ( SHA256STAMP:d1b668fb8b489377151559c908098626bf11550509008b7383f641696582f0ba) diff --git a/doc/lightning-fundpsbt.7.md b/doc/lightning-fundpsbt.7.md index 27ce1b3a9c33..f234fdd50495 100644 --- a/doc/lightning-fundpsbt.7.md +++ b/doc/lightning-fundpsbt.7.md @@ -115,4 +115,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:90e5335b405a3ca2f0a84f3a13044f7f990ae74c29155330f9ba10f4e102f43d) +[comment]: # ( SHA256STAMP:13e35920ba8810db082e3cca62d1141a67498a2756da2479a24eaa62567ff4fe) diff --git a/doc/lightning-getinfo.7.md b/doc/lightning-getinfo.7.md index eb60aee6c789..7d0c237fc665 100644 --- a/doc/lightning-getinfo.7.md +++ b/doc/lightning-getinfo.7.md @@ -132,4 +132,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:d458c44f03d1c242c484627d514b373bf14eca600a0a8390787d370a2ffd2559) +[comment]: # ( SHA256STAMP:043b3816857b0dde57f8233b159f2f932dc72dabd532ca5573b6a0e02b9906d1) diff --git a/doc/lightning-getlog.7.md b/doc/lightning-getlog.7.md index 258e47c23a32..f84c0fc58935 100644 --- a/doc/lightning-getlog.7.md +++ b/doc/lightning-getlog.7.md @@ -95,4 +95,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:398c4068ebf1e340224f86d4eee0d3c7ed326e4b9abbb8f587c0177482f34cf0) +[comment]: # ( SHA256STAMP:5f9808f93c2ec66048455dbda7e59d3afa793559a330a50ba48c3ba66cead7d6) diff --git a/doc/lightning-getroute.7.md b/doc/lightning-getroute.7.md index 09b4ad1af6fb..3c474e045117 100644 --- a/doc/lightning-getroute.7.md +++ b/doc/lightning-getroute.7.md @@ -310,4 +310,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:baddf8e1a08e80c52b89d7a4d03c671d4a34a79eae3ae9c53b8ea6cea8ae5e4d) +[comment]: # ( SHA256STAMP:cc32216cf7ace9054b63870c06de74c2870dc336bf194d3081dab9893cd56f58) diff --git a/doc/lightning-help.7.md b/doc/lightning-help.7.md index 6b8ec9129f88..b8ff2f4bcb46 100644 --- a/doc/lightning-help.7.md +++ b/doc/lightning-help.7.md @@ -69,4 +69,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:bcf64b5073bc6aff09781631cdd9c72c57df5ba84244e59a2b7841f4eb43ab1c) +[comment]: # ( SHA256STAMP:0a8b0e715ffbe4a43bd034485306e54f3eab7f2151532ea3a67fef38fee5932c) diff --git a/doc/lightning-invoice.7.md b/doc/lightning-invoice.7.md index 312ccddca4c3..8635b154cdc4 100644 --- a/doc/lightning-invoice.7.md +++ b/doc/lightning-invoice.7.md @@ -119,4 +119,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:cb98bfe9144b9b0589beaf051da3e3a282103b901771a93c28e76a8e92a43ad5) +[comment]: # ( SHA256STAMP:49fea13084112f78db0e63ad61c1771be94b87d6fa3c34ac77466db638f327cb) diff --git a/doc/lightning-keysend.7.md b/doc/lightning-keysend.7.md index 17309c3df2cf..aba9265d321e 100644 --- a/doc/lightning-keysend.7.md +++ b/doc/lightning-keysend.7.md @@ -118,4 +118,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:0fa28491cf3a3d33ced217e89bd3bdcaa8f0e1b7a547312d19aa8dc3a780161a) +[comment]: # ( SHA256STAMP:41c043374bbe8d916ed6f381efc451c54151ad4de146f397aa0fb317ecbde422) diff --git a/doc/lightning-listchannels.7.md b/doc/lightning-listchannels.7.md index 96e7cc7de227..3bc691c8139a 100644 --- a/doc/lightning-listchannels.7.md +++ b/doc/lightning-listchannels.7.md @@ -80,4 +80,4 @@ Lightning RFC site - BOLT \#7: -[comment]: # ( SHA256STAMP:78f59780528ae5cd33c3607ed11b128cc94e1e0a5e2babd59cb99574a3f5f956) +[comment]: # ( SHA256STAMP:097b3909247d033ccdc82b5b5bbf222ca391140b4fa160c1d5fae714d06c8dce) diff --git a/doc/lightning-listconfigs.7.md b/doc/lightning-listconfigs.7.md index 7d2ca876de3d..ed2a6f6fe13b 100644 --- a/doc/lightning-listconfigs.7.md +++ b/doc/lightning-listconfigs.7.md @@ -222,4 +222,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:2f325aa6ef0506dc627409e3253c8627c49a7d1749ea9dbf24a5baa0fc8c307c) +[comment]: # ( SHA256STAMP:11ff355ba2ee2d5c636cf140f54349a536f3d33554c3ec33fe2a096c0b6fb29c) diff --git a/doc/lightning-listdatastore.7.md b/doc/lightning-listdatastore.7.md index 80d181c8ba1c..4d0cebb0468f 100644 --- a/doc/lightning-listdatastore.7.md +++ b/doc/lightning-listdatastore.7.md @@ -47,4 +47,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:624530c12b1e247c79ee966090bf021f2c8570a8ae73da09cf14302d6784f981) +[comment]: # ( SHA256STAMP:e2b898200862d5b924c6b206f5e168e69cb689ca79610d88039749220b820dd6) diff --git a/doc/lightning-listforwards.7.md b/doc/lightning-listforwards.7.md index 6bdc764c60eb..d129fa3802b9 100644 --- a/doc/lightning-listforwards.7.md +++ b/doc/lightning-listforwards.7.md @@ -64,4 +64,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:68d847297711c3881fc9118de8e0e3994b7f4fb2a831b62dca1bb33d490d8416) +[comment]: # ( SHA256STAMP:10b0eea0c6b65287a913ce0c4c4d73d089e2e45d81e5c360f345bee950db0957) diff --git a/doc/lightning-listfunds.7.md b/doc/lightning-listfunds.7.md index 8ac30af7c36d..52d181ec44da 100644 --- a/doc/lightning-listfunds.7.md +++ b/doc/lightning-listfunds.7.md @@ -73,4 +73,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:a9a29cb20e610f7c62f61bba5826d76f4f6be72b1e8f0fecfb8207c1b90712a1) +[comment]: # ( SHA256STAMP:d420be5d3e7fa9cf8b21ac928b91ebe08bb68a782e2a04b21b215d81066094f5) diff --git a/doc/lightning-listhtlcs.7.md b/doc/lightning-listhtlcs.7.md index 82d0019a01a4..e85f84ff211a 100644 --- a/doc/lightning-listhtlcs.7.md +++ b/doc/lightning-listhtlcs.7.md @@ -46,4 +46,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:990e36b109c9e318bc566a951ce0d39032e252cdd1555c75ad7b168d547c937f) +[comment]: # ( SHA256STAMP:55c907e8a82cde84d1bda26c453d393e6e3e8383e89256a4efce1cb4de4bcbb6) diff --git a/doc/lightning-listinvoices.7.md b/doc/lightning-listinvoices.7.md index 7a4ff82de7bb..948c410882e0 100644 --- a/doc/lightning-listinvoices.7.md +++ b/doc/lightning-listinvoices.7.md @@ -58,4 +58,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:0dd6207e711b96094310c9d6b56575eb7e475a4a5bf728cd2b6e8d408ed3abbe) +[comment]: # ( SHA256STAMP:651c927eb891cd7110124d7a3c472e64fb24299bd3f8216f7cdd7880f8c8821f) diff --git a/doc/lightning-listnodes.7.md b/doc/lightning-listnodes.7.md index a86f3ab3d815..db6cf382b5e5 100644 --- a/doc/lightning-listnodes.7.md +++ b/doc/lightning-listnodes.7.md @@ -100,4 +100,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:938937f92ed57a3ff70ae5613ccc1709d500ea91c77ca261cb82f9f40c30579e) +[comment]: # ( SHA256STAMP:99d22f32acd7d5181f731342d7b9245cfa17cc4257c1f87d78eb809fe7c6931d) diff --git a/doc/lightning-listoffers.7.md b/doc/lightning-listoffers.7.md index ad58ae4fa3e4..fe6c8ea68430 100644 --- a/doc/lightning-listoffers.7.md +++ b/doc/lightning-listoffers.7.md @@ -81,4 +81,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:e3e1446c2c1bf74852663c1d417bb386908c47898b05e958f153f806e73a9b4c) +[comment]: # ( SHA256STAMP:b968ff5dd2eb57cb030c7a9fb73d3d305e69f23dc2ccd599573fb345e0f58385) diff --git a/doc/lightning-listpays.7.md b/doc/lightning-listpays.7.md index 6477f4ce002f..44dfa66fb44e 100644 --- a/doc/lightning-listpays.7.md +++ b/doc/lightning-listpays.7.md @@ -57,4 +57,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ebdcb353fc2b11f6d83610c538206ef348a24b77af4cb2dcd535235f50ee5c39) +[comment]: # ( SHA256STAMP:adf164794675b251cbef10f5c40dd2e05812fc681641e0631126bae2a8bc8883) diff --git a/doc/lightning-listpeerchannels.7.md b/doc/lightning-listpeerchannels.7.md index d28fb27f15f5..f8c5ab6c7217 100644 --- a/doc/lightning-listpeerchannels.7.md +++ b/doc/lightning-listpeerchannels.7.md @@ -191,4 +191,4 @@ Main web site: Lightning RFC site (BOLT \#9): -[comment]: # ( SHA256STAMP:485344d9d9a47bf3093c8e54fcf80bea14623e2611f2a4e2d2b5c723d8e6094b) +[comment]: # ( SHA256STAMP:64f0c002713473b6b8e06fc6b5eba0dc830b98df21f8aa5be9a80052c5b7a7e7) diff --git a/doc/lightning-listpeers.7.md b/doc/lightning-listpeers.7.md index dedf608603a8..a3301345057f 100644 --- a/doc/lightning-listpeers.7.md +++ b/doc/lightning-listpeers.7.md @@ -399,4 +399,4 @@ Main web site: Lightning RFC site (BOLT \#9): -[comment]: # ( SHA256STAMP:a063a4a4fb1e6af4138d19ccbdc8d1539712c6eb6ed8a4b1b7f7c4fe12e0907b) +[comment]: # ( SHA256STAMP:696c0d20a0fdc2a40531be5f38a9460272c4ff70667433fd08da3309b74286d5) diff --git a/doc/lightning-listsendpays.7.md b/doc/lightning-listsendpays.7.md index 52ddbd108db8..3070683ff2e3 100644 --- a/doc/lightning-listsendpays.7.md +++ b/doc/lightning-listsendpays.7.md @@ -65,4 +65,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:a6dcb3708d706650f74fcdd4d05614a813ac5a69c13c4c579d45c01b106545e2) +[comment]: # ( SHA256STAMP:c00443620140062eadf4ccdc32f08102d8278c80975437a708e1b4913b6ab85a) diff --git a/doc/lightning-listtransactions.7.md b/doc/lightning-listtransactions.7.md index a0f8e72bd12e..ea5a71228c56 100644 --- a/doc/lightning-listtransactions.7.md +++ b/doc/lightning-listtransactions.7.md @@ -103,4 +103,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:4820c0c2f399fd5bec1a960bdc731c131a0fb019f7506df3053ae1bc08705845) +[comment]: # ( SHA256STAMP:b3b7286a81cdae413baac015e87d65a095506accb32ecea5dc1fdccdc8e73c4c) diff --git a/doc/lightning-makesecret.7.md b/doc/lightning-makesecret.7.md index baf8f159ac48..bec1269a5b95 100644 --- a/doc/lightning-makesecret.7.md +++ b/doc/lightning-makesecret.7.md @@ -38,4 +38,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:3a09136243f716a657368f4483d4211ac37853eb69f26b8b30f1270ed2c6e993) +[comment]: # ( SHA256STAMP:3c8977e48a221ea5354f1affb1ae86b606aca2df4af32bad0a101386a12556a3) diff --git a/doc/lightning-multifundchannel.7.md b/doc/lightning-multifundchannel.7.md index fdb456907c31..2c6b63a02376 100644 --- a/doc/lightning-multifundchannel.7.md +++ b/doc/lightning-multifundchannel.7.md @@ -164,4 +164,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:604a53a621512eebeeccdd2ec677f08577b3cd3c41d10439a7eb00ae1fe11121) +[comment]: # ( SHA256STAMP:9922effdfb4bcd5ab95057fb0c043f0597446f4da4e7d5033520a3138ffc8ff8) diff --git a/doc/lightning-multiwithdraw.7.md b/doc/lightning-multiwithdraw.7.md index 7dd8c377c52c..ae09c2bdf251 100644 --- a/doc/lightning-multiwithdraw.7.md +++ b/doc/lightning-multiwithdraw.7.md @@ -73,4 +73,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:715042b3e709227dcb00068190a8566a1fe678840aa56e566fd4af27971150c8) +[comment]: # ( SHA256STAMP:3a090511614bdae6c1160609bb4b8ec35d4ca81dbfc9fc5a6c3f3b70afc19a1d) diff --git a/doc/lightning-newaddr.7.md b/doc/lightning-newaddr.7.md index 3de7419b4af7..902e326b71d3 100644 --- a/doc/lightning-newaddr.7.md +++ b/doc/lightning-newaddr.7.md @@ -56,4 +56,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:9d8dc613c005127a0807f2c8b26b0a96ddc5bf3ebdfa59c3f95a888476c0ce2a) +[comment]: # ( SHA256STAMP:90d550bc2290dd2ab6ee67e377679fe45230a14ba6f4608fda8e51bb6670cc07) diff --git a/doc/lightning-notifications.7.md b/doc/lightning-notifications.7.md index 69ffba4ab52f..fd06e15b968b 100644 --- a/doc/lightning-notifications.7.md +++ b/doc/lightning-notifications.7.md @@ -103,4 +103,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ec98523e094209b75eeeb620d8f2a64409dafe6ba21baf3a89ade514b285d202) +[comment]: # ( SHA256STAMP:41d0ca6a956520453538c8ad5c5afce681540f4ce26017570cdc2356c3aab599) diff --git a/doc/lightning-offer.7.md b/doc/lightning-offer.7.md index d2b4ed5e0056..24d7fa36f365 100644 --- a/doc/lightning-offer.7.md +++ b/doc/lightning-offer.7.md @@ -131,4 +131,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ea0d06ff5697d92e37b9c737324ed0c10417b5fc2374cde2590c59c48821bc88) +[comment]: # ( SHA256STAMP:46e6bfcfe48f709e2f2068250e28b53bedafb782db42791e722d301515fce070) diff --git a/doc/lightning-openchannel_abort.7.md b/doc/lightning-openchannel_abort.7.md index efe862f7cdbd..0878c87a72d3 100644 --- a/doc/lightning-openchannel_abort.7.md +++ b/doc/lightning-openchannel_abort.7.md @@ -56,4 +56,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:c6f95d7c5638e19315ae917ea64adce77cfa492c12be843813a93e03027c82f1) +[comment]: # ( SHA256STAMP:51ed12ef563f25e818645df9d84a70d409f2dc0404d4ec2f754f0bbadbc06a52) diff --git a/doc/lightning-openchannel_bump.7.md b/doc/lightning-openchannel_bump.7.md index a284959f64bf..09af64fd1f7e 100644 --- a/doc/lightning-openchannel_bump.7.md +++ b/doc/lightning-openchannel_bump.7.md @@ -82,4 +82,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:346ffc931abfac76cb31643ee032fc8fbdc0ab164bf2862d51a8e33944bbfa86) +[comment]: # ( SHA256STAMP:ed3aa14a604515d218f9a15dd02997a055effc5cb38b52a111466fb44ab06198) diff --git a/doc/lightning-openchannel_init.7.md b/doc/lightning-openchannel_init.7.md index 914aac45d0ff..475337d90f89 100644 --- a/doc/lightning-openchannel_init.7.md +++ b/doc/lightning-openchannel_init.7.md @@ -104,4 +104,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:d8d6bf454bb96dbb679e37ca7d1f13789a78653dced75c840433091b6cc6ffed) +[comment]: # ( SHA256STAMP:d957b5bb745977f93805ddd65943e74acbdc68b01ebd5bb2f13ef2b24463b859) diff --git a/doc/lightning-openchannel_signed.7.md b/doc/lightning-openchannel_signed.7.md index 4ce5919fc481..e80c37091425 100644 --- a/doc/lightning-openchannel_signed.7.md +++ b/doc/lightning-openchannel_signed.7.md @@ -68,4 +68,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:61131b40f71d7c11a7a4d1633cf57a4a14bd60ff6b8867f06183bee60569ef67) +[comment]: # ( SHA256STAMP:694c288e5a49a662b2b7d01cbe46b6c0c024242bd1745b20e3a1eae123e569fe) diff --git a/doc/lightning-openchannel_update.7.md b/doc/lightning-openchannel_update.7.md index 1fbd9428ac17..d337656191b4 100644 --- a/doc/lightning-openchannel_update.7.md +++ b/doc/lightning-openchannel_update.7.md @@ -73,4 +73,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:0790e67080591d43ff7dcb033bef7e96d04be6719aa22deec69f2e019634c48d) +[comment]: # ( SHA256STAMP:11e23b688eb714707cf3203397761454b140a96ab5d7512208013700227aff4c) diff --git a/doc/lightning-parsefeerate.7.md b/doc/lightning-parsefeerate.7.md index d69024aae890..581d8376cba6 100644 --- a/doc/lightning-parsefeerate.7.md +++ b/doc/lightning-parsefeerate.7.md @@ -44,4 +44,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:d192483fc5442582e9765fcb0657f3470083a1acfffdcbf72b9e414581a7f0e3) +[comment]: # ( SHA256STAMP:be616b76a92bb1d8863350cf44b79f9d2cd8a6e9a7993bd9b5e704d9e0038790) diff --git a/doc/lightning-pay.7.md b/doc/lightning-pay.7.md index 5a6e0f909e3c..280bae16dd4e 100644 --- a/doc/lightning-pay.7.md +++ b/doc/lightning-pay.7.md @@ -168,4 +168,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:772f97fc664c5c6ca4cc24eade35b82d55681732518351e6343caed25bd7c2b5) +[comment]: # ( SHA256STAMP:6e644d1071a3e0b9f53e67ced3b29702d93849402e3e6893f41db9f4b23da064) diff --git a/doc/lightning-ping.7.md b/doc/lightning-ping.7.md index 32e4afe92c62..dd0af4510879 100644 --- a/doc/lightning-ping.7.md +++ b/doc/lightning-ping.7.md @@ -71,4 +71,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:81b2e86262fb39f609f2ea93ffa2b3236758759d51612d331bbcc87406a15637) +[comment]: # ( SHA256STAMP:7fe1120c251ffe6d51057a94823376a512dee3ec4f251be82a7dc4b2f044a165) diff --git a/doc/lightning-plugin.7.md b/doc/lightning-plugin.7.md index 9a1c8ba7584f..a9311c651534 100644 --- a/doc/lightning-plugin.7.md +++ b/doc/lightning-plugin.7.md @@ -84,4 +84,4 @@ RESOURCES Main web site: [writing plugins]: PLUGINS.md -[comment]: # ( SHA256STAMP:243a25cee50c6f04262551bc170f2c6b1d123ba9e85d574a2facaa2aabed5dc9) +[comment]: # ( SHA256STAMP:66b5b924fa927c85e065fd01a7b94a0a892b3e027830a8c1f2c584586ee2a7e7) diff --git a/doc/lightning-preapproveinvoice.7.md b/doc/lightning-preapproveinvoice.7.md index e4ff52291d7c..1af6b171766e 100644 --- a/doc/lightning-preapproveinvoice.7.md +++ b/doc/lightning-preapproveinvoice.7.md @@ -48,4 +48,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ec98523e094209b75eeeb620d8f2a64409dafe6ba21baf3a89ade514b285d202) +[comment]: # ( SHA256STAMP:41d0ca6a956520453538c8ad5c5afce681540f4ce26017570cdc2356c3aab599) diff --git a/doc/lightning-preapprovekeysend.7.md b/doc/lightning-preapprovekeysend.7.md index 0f0f776f37d0..05e928887d0b 100644 --- a/doc/lightning-preapprovekeysend.7.md +++ b/doc/lightning-preapprovekeysend.7.md @@ -58,4 +58,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ec98523e094209b75eeeb620d8f2a64409dafe6ba21baf3a89ade514b285d202) +[comment]: # ( SHA256STAMP:41d0ca6a956520453538c8ad5c5afce681540f4ce26017570cdc2356c3aab599) diff --git a/doc/lightning-reserveinputs.7.md b/doc/lightning-reserveinputs.7.md index 6731214ac751..d0407361f161 100644 --- a/doc/lightning-reserveinputs.7.md +++ b/doc/lightning-reserveinputs.7.md @@ -64,4 +64,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ab791403170230278c7b17a81af79d7b5332188a4f23c83c81dfa97638584e5e) +[comment]: # ( SHA256STAMP:0df4568eb6977f3837270b935c26792bc08d18cfa05ce0c517aae880cf0b497b) diff --git a/doc/lightning-sendcustommsg.7.md b/doc/lightning-sendcustommsg.7.md index 08876225958e..43bfeeefc973 100644 --- a/doc/lightning-sendcustommsg.7.md +++ b/doc/lightning-sendcustommsg.7.md @@ -69,4 +69,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:66216a842041c93d3527d7130dbb57fd4b34e6b5426fdc1bfd368302f6f8c611) +[comment]: # ( SHA256STAMP:0f455705de4f2f2e3d4ed8471ec3d0bf77865d8cf769884fe2b5eca40879fcaa) diff --git a/doc/lightning-sendinvoice.7.md b/doc/lightning-sendinvoice.7.md index d5253310d4ec..69610cb57507 100644 --- a/doc/lightning-sendinvoice.7.md +++ b/doc/lightning-sendinvoice.7.md @@ -80,4 +80,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:9e35052033fbb78416b024c22d8ecca256eaaf45b16b3b23b09a14920c155133) +[comment]: # ( SHA256STAMP:a088e5202cb822518a2214b485083edf0a929290b17beb9a3930dd93d97411d0) diff --git a/doc/lightning-sendonion.7.md b/doc/lightning-sendonion.7.md index ec1f0cdc5dbe..051edc79e8c1 100644 --- a/doc/lightning-sendonion.7.md +++ b/doc/lightning-sendonion.7.md @@ -135,4 +135,4 @@ RESOURCES Main web site: [bolt04]: https://github.com/lightning/bolts/blob/master/04-onion-routing.md -[comment]: # ( SHA256STAMP:dab3d2111fac44ea7d0183485e6f67566d2c577f9dafde8a897f438fab04e7e6) +[comment]: # ( SHA256STAMP:88fc091802bfa0295ce3b2e445160d2357a66d6501328bdb086498960b2f915d) diff --git a/doc/lightning-sendonionmessage.7.md b/doc/lightning-sendonionmessage.7.md index d7c700f4e8ed..31abab740333 100644 --- a/doc/lightning-sendonionmessage.7.md +++ b/doc/lightning-sendonionmessage.7.md @@ -43,4 +43,4 @@ Main web site: [bolt04]: https://github.com/lightning/bolts/blob/master/04-onion-routing.md -[comment]: # ( SHA256STAMP:a2b84b83e10b81fd82a2bed20874f707de6002c076a4276d4f6ff30772ad3e88) +[comment]: # ( SHA256STAMP:636acc798ed7ae1cd307ada4dbde424c1ed8aa514600bec9adeacd5778f4d036) diff --git a/doc/lightning-sendpay.7.md b/doc/lightning-sendpay.7.md index 0a63a09c7a74..bb2271b92938 100644 --- a/doc/lightning-sendpay.7.md +++ b/doc/lightning-sendpay.7.md @@ -142,4 +142,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:d5df663428080fdd470950ec3c0c6630511f0936e7acd3ed860313ed95f33579) +[comment]: # ( SHA256STAMP:50657fd6d5bef4f88ae1d38c5ddfe0d42eb67f222d51e90b03354c68c56f7905) diff --git a/doc/lightning-sendpsbt.7.md b/doc/lightning-sendpsbt.7.md index cdc7aabe906e..a536aa349b14 100644 --- a/doc/lightning-sendpsbt.7.md +++ b/doc/lightning-sendpsbt.7.md @@ -67,4 +67,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:715042b3e709227dcb00068190a8566a1fe678840aa56e566fd4af27971150c8) +[comment]: # ( SHA256STAMP:3a090511614bdae6c1160609bb4b8ec35d4ca81dbfc9fc5a6c3f3b70afc19a1d) diff --git a/doc/lightning-setchannel.7.md b/doc/lightning-setchannel.7.md index 0ef80772895e..199163460a8d 100644 --- a/doc/lightning-setchannel.7.md +++ b/doc/lightning-setchannel.7.md @@ -107,4 +107,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:3c0f2eb8fc8cfcebea463eaefab17eb96310b24ce6566f5c5a8f95bebf81fcb4) +[comment]: # ( SHA256STAMP:0175aaf74aa6d75640d0b79f68b552b1b700f93ad7576465c73de93af04d71e6) diff --git a/doc/lightning-signmessage.7.md b/doc/lightning-signmessage.7.md index b05b3026f9cb..88b7fc5a4806 100644 --- a/doc/lightning-signmessage.7.md +++ b/doc/lightning-signmessage.7.md @@ -42,4 +42,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:7b56f693aa33a88cf38459ff1581dd0fc09c1280b1068530e607f73ee62d4266) +[comment]: # ( SHA256STAMP:04bac6c24dea9dbf1f0d42015b1452875154d4f270e264fbad5145ed4b747448) diff --git a/doc/lightning-signpsbt.7.md b/doc/lightning-signpsbt.7.md index ff5ffc9db600..f0809ccd15a7 100644 --- a/doc/lightning-signpsbt.7.md +++ b/doc/lightning-signpsbt.7.md @@ -73,4 +73,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:83b5bc1c0d33cdf5bae731ab6ead5aef3aff32c09a13902ebb9b96429d02dca5) +[comment]: # ( SHA256STAMP:d016f1aab17aab7a4d2f127cbab4af79a3b43f35a5e6f893ddd355110520e111) diff --git a/doc/lightning-stop.7.md b/doc/lightning-stop.7.md index 67e4e61ef545..a09e67103d24 100644 --- a/doc/lightning-stop.7.md +++ b/doc/lightning-stop.7.md @@ -43,4 +43,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:e119fe5893a4efe640aba23062bf2a37797ea8071293179c40398b824b1446cd) +[comment]: # ( SHA256STAMP:9ae0ce5a61e36232d45cf5d8bb6a84b7fdff4137fadfdcd5a35fdf995ce8ad84) diff --git a/doc/lightning-txdiscard.7.md b/doc/lightning-txdiscard.7.md index f3fc1a94bc64..03aa39f2f320 100644 --- a/doc/lightning-txdiscard.7.md +++ b/doc/lightning-txdiscard.7.md @@ -45,4 +45,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:adcbdf53ef9b0ed3c60098e70555e51e94f26c6128cbc9a54307ef75a9eabd2a) +[comment]: # ( SHA256STAMP:ce0f0a09f198650085f8877ef51a9fa9df6cdf8aed109512e0ea6bda33628bd2) diff --git a/doc/lightning-txprepare.7.md b/doc/lightning-txprepare.7.md index 5ba2f41439c9..37655098c771 100644 --- a/doc/lightning-txprepare.7.md +++ b/doc/lightning-txprepare.7.md @@ -85,4 +85,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:376f1c7c33637cccb1c65304063251a792059914d2059e376fcf3d52a2e5c469) +[comment]: # ( SHA256STAMP:ca40d2eaea3ecd2f3e27ec879d09fe73600fa17d15b098abc8030ac320ec9c4e) diff --git a/doc/lightning-txsend.7.md b/doc/lightning-txsend.7.md index f85bda7eb348..1953f25dd80a 100644 --- a/doc/lightning-txsend.7.md +++ b/doc/lightning-txsend.7.md @@ -45,4 +45,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:dd7fb4170a0507e6097ef8ba78fb937902604d5814b3fb8ce15c5f579d3e15e8) +[comment]: # ( SHA256STAMP:bce892d19609ab19255db773e01eee1caac19481b4d4f8af3ffd5b148d120157) diff --git a/doc/lightning-unreserveinputs.7.md b/doc/lightning-unreserveinputs.7.md index f86e039a9378..5096b25944c5 100644 --- a/doc/lightning-unreserveinputs.7.md +++ b/doc/lightning-unreserveinputs.7.md @@ -55,4 +55,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:9f00af2719fcbddbffd40d57d0cddc4d8f8c4ba3855aee63f7ee796f1828e9b9) +[comment]: # ( SHA256STAMP:c8b09a8971d97627d242e348c13d38671e84467a7afa1dc0a73941ab13fdeaff) diff --git a/doc/lightning-utxopsbt.7.md b/doc/lightning-utxopsbt.7.md index a05441a96f3e..5744081968e0 100644 --- a/doc/lightning-utxopsbt.7.md +++ b/doc/lightning-utxopsbt.7.md @@ -100,4 +100,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:12ae2b4f71606d6bb3972e3259ad27fc4369a72482cff3db28537ae9cdad4817) +[comment]: # ( SHA256STAMP:818cccd0ff2ed3398ceb036dd4034484f965220844de916a846cd6cf17a14fd3) diff --git a/doc/lightning-waitanyinvoice.7.md b/doc/lightning-waitanyinvoice.7.md index 394100409498..c48d4fbc4ce4 100644 --- a/doc/lightning-waitanyinvoice.7.md +++ b/doc/lightning-waitanyinvoice.7.md @@ -75,4 +75,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:332f6ef658d50377b3e7b9ee2f5583dfbaf5034c7403d0b6ca8167295a9255e3) +[comment]: # ( SHA256STAMP:846510edabc52b21c0ae6482a49e373ebe7feeb4697b6a6f48d85b30351086f2) diff --git a/doc/lightning-waitblockheight.7.md b/doc/lightning-waitblockheight.7.md index f1ea70caf535..7e4ec8ee3e63 100644 --- a/doc/lightning-waitblockheight.7.md +++ b/doc/lightning-waitblockheight.7.md @@ -39,4 +39,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:b8c8ba9b75da521171fcfa0c8dc4a1fdeb271e2c4dfacd445223aeda06910141) +[comment]: # ( SHA256STAMP:2b85d1114720e3bf8a2b3060aefc00faa89fdcf52b61ab5ff11cd273f1799fba) diff --git a/doc/lightning-waitinvoice.7.md b/doc/lightning-waitinvoice.7.md index b5bd8309a349..254c24fb4aca 100644 --- a/doc/lightning-waitinvoice.7.md +++ b/doc/lightning-waitinvoice.7.md @@ -60,4 +60,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:332f6ef658d50377b3e7b9ee2f5583dfbaf5034c7403d0b6ca8167295a9255e3) +[comment]: # ( SHA256STAMP:846510edabc52b21c0ae6482a49e373ebe7feeb4697b6a6f48d85b30351086f2) diff --git a/doc/lightning-waitsendpay.7.md b/doc/lightning-waitsendpay.7.md index 1df5b4819f05..f441be795ad3 100644 --- a/doc/lightning-waitsendpay.7.md +++ b/doc/lightning-waitsendpay.7.md @@ -104,4 +104,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:f7316b6ec1f7f0e4b652915baf3dba919a8c9f64f274b142429801809fbacf8a) +[comment]: # ( SHA256STAMP:6131f1442571e836e38b1570e75639c3d499b4602659a19ad03aa7156d58d949) diff --git a/doc/lightning-withdraw.7.md b/doc/lightning-withdraw.7.md index c58271ac8ef5..d5e188d09657 100644 --- a/doc/lightning-withdraw.7.md +++ b/doc/lightning-withdraw.7.md @@ -74,4 +74,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:57a5a6255500599f39425df668368601f24c95e09b7f0174662d7fba54f05374) +[comment]: # ( SHA256STAMP:38527c3337263c9b4681c976a8148acaaa544f94beb576f2a91b584c3488bfc3) diff --git a/tools/fromschema.py b/tools/fromschema.py index 1e78ef4f62ef..e312ac617642 100755 --- a/tools/fromschema.py +++ b/tools/fromschema.py @@ -260,7 +260,7 @@ def generate_from_schema(schema): # Don't have a description field here, it's not used. assert 'description' not in toplevels[0] sub = props[toplevels[0]] - elif len(toplevels) == 1 and props[toplevels[0]]['type'] == 'array': + elif len(toplevels) == 1 and props[toplevels[0]]['type'] == 'array' and props[toplevels[0]]['items']['type'] == 'object': output('On success, an object containing {} is returned. It is an array of objects, where each object contains:\n\n'.format(fmt_propname(toplevels[0]))) # Don't have a description field here, it's not used. assert 'description' not in toplevels[0] From 4418172c985f4dbe6cd9e282216a5fab1e99d188 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:16 +1030 Subject: [PATCH 17/36] common: add routine to get double from JSON. I don't like it, but we do expose some times like this :( Signed-off-by: Rusty Russell --- common/json_parse_simple.c | 16 ++++++++++++++++ common/json_parse_simple.h | 3 +++ 2 files changed, 19 insertions(+) diff --git a/common/json_parse_simple.c b/common/json_parse_simple.c index 236c89befebe..0ccbd9573a5c 100644 --- a/common/json_parse_simple.c +++ b/common/json_parse_simple.c @@ -101,6 +101,22 @@ bool json_str_to_u64(const char *buffer, const jsmntok_t *tok, u64 *num) return json_to_u64(buffer, &temp, num); } +bool json_to_double(const char *buffer, const jsmntok_t *tok, double *num) +{ + char *end; + + errno = 0; + *num = strtod(buffer + tok->start, &end); + if (end != buffer + tok->end) + return false; + + /* Check for overflow */ + if (errno == ERANGE) + return false; + + return true; +} + bool json_to_u32(const char *buffer, const jsmntok_t *tok, u32 *num) { uint64_t u64; diff --git a/common/json_parse_simple.h b/common/json_parse_simple.h index 697dd4bade71..2c7c3c8975a6 100644 --- a/common/json_parse_simple.h +++ b/common/json_parse_simple.h @@ -42,6 +42,9 @@ bool json_str_to_u64(const char *buffer, const jsmntok_t *tok, u64 *num); /* Extract number from this (may be a string, or a number literal) */ bool json_to_u32(const char *buffer, const jsmntok_t *tok, u32 *num); +/* Extract double from this (generally a bad idea!) */ +bool json_to_double(const char *buffer, const jsmntok_t *tok, double *num); + /* Extract boolean from this */ bool json_to_bool(const char *buffer, const jsmntok_t *tok, bool *b); From 1f2f1b72046b5f1778e967b25a9facc9c88d33a7 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:16 +1030 Subject: [PATCH 18/36] doc/schemas: remove unnecessary length restrictions. If we have a specific type (not just "hex") the length is implied. Signed-off-by: Rusty Russell --- doc/lightning-createinvoice.7.md | 6 +++--- doc/lightning-createonion.7.md | 4 ++-- doc/lightning-delinvoice.7.md | 6 +++--- doc/lightning-invoice.7.md | 6 +++--- doc/lightning-keysend.7.md | 6 +++--- doc/lightning-listinvoices.7.md | 6 +++--- doc/lightning-listpeerchannels.7.md | 6 +++--- doc/lightning-listsendpays.7.md | 6 +++--- doc/lightning-makesecret.7.md | 4 ++-- doc/lightning-pay.7.md | 6 +++--- doc/lightning-sendonion.7.md | 6 +++--- doc/lightning-sendpay.7.md | 6 +++--- doc/lightning-waitanyinvoice.7.md | 6 +++--- doc/lightning-waitinvoice.7.md | 6 +++--- doc/lightning-waitsendpay.7.md | 6 +++--- doc/schemas/createinvoice.schema.json | 8 ++------ doc/schemas/createonion.schema.json | 4 +--- doc/schemas/delinvoice.schema.json | 8 ++------ doc/schemas/invoice.schema.json | 8 ++------ doc/schemas/keysend.schema.json | 8 ++------ doc/schemas/listinvoices.schema.json | 8 ++------ doc/schemas/listpeerchannels.schema.json | 8 ++------ doc/schemas/listsendpays.schema.json | 8 ++------ doc/schemas/makesecret.schema.json | 4 +--- doc/schemas/pay.schema.json | 8 ++------ doc/schemas/sendonion.schema.json | 8 ++------ doc/schemas/sendpay.schema.json | 8 ++------ doc/schemas/waitanyinvoice.schema.json | 8 ++------ doc/schemas/waitinvoice.schema.json | 8 ++------ doc/schemas/waitsendpay.schema.json | 8 ++------ 30 files changed, 71 insertions(+), 127 deletions(-) diff --git a/doc/lightning-createinvoice.7.md b/doc/lightning-createinvoice.7.md index 2652d8dc7503..f841a7f95775 100644 --- a/doc/lightning-createinvoice.7.md +++ b/doc/lightning-createinvoice.7.md @@ -34,7 +34,7 @@ RETURN VALUE On success, an object is returned, containing: - **label** (string): the label for the invoice -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): Whether it has been paid, or can no longer be paid (one of "paid", "expired", "unpaid") - **description** (string): Description extracted from **bolt11** or **bolt12** - **expires\_at** (u64): UNIX timestamp of when invoice expires (or expired) @@ -44,7 +44,7 @@ On success, an object is returned, containing: - **pay\_index** (u64, optional): Incrementing id for when this was paid (**status** *paid* only) - **amount\_received\_msat** (msat, optional): Amount actually received (**status** *paid* only) - **paid\_at** (u64, optional): UNIX timestamp of when invoice was paid (**status** *paid* only) -- **payment\_preimage** (secret, optional): the proof of payment: SHA256 of this **payment\_hash** (always 64 characters) +- **payment\_preimage** (secret, optional): the proof of payment: SHA256 of this **payment\_hash** - **local\_offer\_id** (hex, optional): the *id* of our offer which created this invoice (**experimental-offers** only). (always 64 characters) - **invreq\_payer\_note** (string, optional): the optional *invreq\_payer\_note* from invoice\_request which created this invoice (**experimental-offers** only). @@ -75,4 +75,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:1da48df402e8ea4fa1e5901d3272dc9d09f32fd62b03cf90393778add281b732) +[comment]: # ( SHA256STAMP:fffebe36aa6671261082894e8b1429035c08f797064a60b03e3e9ea10ea71038) diff --git a/doc/lightning-createonion.7.md b/doc/lightning-createonion.7.md index b240257f1e64..cc599867fe5f 100644 --- a/doc/lightning-createonion.7.md +++ b/doc/lightning-createonion.7.md @@ -93,7 +93,7 @@ On success, an object is returned, containing: - **onion** (hex): the onion packet (*onion\_size* bytes) - **shared\_secrets** (array of secrets): one shared secret for each node in the *hops* parameter: - - the shared secret with this hop (always 64 characters) + - the shared secret with this hop [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -132,4 +132,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:31882493866e2d3fea36ab68e610637609967fc1095bc926da9bdfbd578de08b) +[comment]: # ( SHA256STAMP:08d376f24ca65df41645bd82fa8c8d19fa8610fb5e41f252f001845334b68fbb) diff --git a/doc/lightning-delinvoice.7.md b/doc/lightning-delinvoice.7.md index 134c9a2a6c90..69b2b3550efb 100644 --- a/doc/lightning-delinvoice.7.md +++ b/doc/lightning-delinvoice.7.md @@ -28,7 +28,7 @@ Note: The return is the same as an object from lightning-listinvoice(7). On success, an object is returned, containing: - **label** (string): Unique label given at creation time -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): State of invoice (one of "paid", "expired", "unpaid") - **expires\_at** (u64): UNIX timestamp when invoice expires (or expired) - **bolt11** (string, optional): BOLT11 string @@ -46,7 +46,7 @@ If **status** is "paid": - **pay\_index** (u64): unique index for this invoice payment - **amount\_received\_msat** (msat): how much was actually received - **paid\_at** (u64): UNIX timestamp of when payment was received - - **payment\_preimage** (secret): SHA256 of this is the *payment\_hash* offered in the invoice (always 64 characters) + - **payment\_preimage** (secret): SHA256 of this is the *payment\_hash* offered in the invoice [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -81,4 +81,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:453b05959f4f08b5d7f09d2ba6f5504d8775d3e8a81b7e02b20f755dfe3ded88) +[comment]: # ( SHA256STAMP:7135589b2dd43b221132efed8753afd2e3ecc0aa9ad5706753fbd6c5b54c1509) diff --git a/doc/lightning-invoice.7.md b/doc/lightning-invoice.7.md index 8635b154cdc4..f6d549cafa17 100644 --- a/doc/lightning-invoice.7.md +++ b/doc/lightning-invoice.7.md @@ -79,8 +79,8 @@ RETURN VALUE On success, an object is returned, containing: - **bolt11** (string): the bolt11 string -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) -- **payment\_secret** (secret): the *payment\_secret* to place in the onion (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment +- **payment\_secret** (secret): the *payment\_secret* to place in the onion - **expires\_at** (u64): UNIX timestamp of when invoice expires The following warnings may also be returned: @@ -119,4 +119,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:49fea13084112f78db0e63ad61c1771be94b87d6fa3c34ac77466db638f327cb) +[comment]: # ( SHA256STAMP:095393c4a1050a9a458eba1033162e99283019329747a66b6461a5bb13fa7a2f) diff --git a/doc/lightning-keysend.7.md b/doc/lightning-keysend.7.md index aba9265d321e..60e22428c0b3 100644 --- a/doc/lightning-keysend.7.md +++ b/doc/lightning-keysend.7.md @@ -70,8 +70,8 @@ RETURN VALUE [comment]: # (GENERATE-FROM-SCHEMA-START) On success, an object is returned, containing: -- **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** (always 64 characters) -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **created\_at** (number): the UNIX timestamp showing when this payment was initiated - **parts** (u32): how many attempts this took - **amount\_msat** (msat): Amount the recipient received @@ -118,4 +118,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:41c043374bbe8d916ed6f381efc451c54151ad4de146f397aa0fb317ecbde422) +[comment]: # ( SHA256STAMP:388b5d185f053b32176eef14bc659c405f56b4096b7635d2eac38583b0285889) diff --git a/doc/lightning-listinvoices.7.md b/doc/lightning-listinvoices.7.md index 948c410882e0..a2d4e41eacc8 100644 --- a/doc/lightning-listinvoices.7.md +++ b/doc/lightning-listinvoices.7.md @@ -24,7 +24,7 @@ RETURN VALUE On success, an object containing **invoices** is returned. It is an array of objects, where each object contains: - **label** (string): unique label supplied at invoice creation -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): Whether it's paid, unpaid or unpayable (one of "unpaid", "paid", "expired") - **expires\_at** (u64): UNIX timestamp of when it will become / became unpayable - **description** (string, optional): description used in the invoice @@ -39,7 +39,7 @@ If **status** is "paid": - **pay\_index** (u64): Unique incrementing index for this payment - **amount\_received\_msat** (msat): the amount actually received (could be slightly greater than *amount\_msat*, since clients may overpay) - **paid\_at** (u64): UNIX timestamp of when it was paid - - **payment\_preimage** (secret): proof of payment (always 64 characters) + - **payment\_preimage** (secret): proof of payment [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -58,4 +58,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:651c927eb891cd7110124d7a3c472e64fb24299bd3f8216f7cdd7880f8c8821f) +[comment]: # ( SHA256STAMP:258d690221dce5a8811f361e59a5c0059190f140e420c65cd37cfd0987efea3a) diff --git a/doc/lightning-listpeerchannels.7.md b/doc/lightning-listpeerchannels.7.md index f8c5ab6c7217..cc3e59f2b2cd 100644 --- a/doc/lightning-listpeerchannels.7.md +++ b/doc/lightning-listpeerchannels.7.md @@ -36,7 +36,7 @@ On success, an object containing **channels** is returned. It is an array of ob - **perkb** (u32): Feerate per 1000 virtual bytes - **owner** (string, optional): The current subdaemon controlling this connection - **short\_channel\_id** (short\_channel\_id, optional): The short\_channel\_id (once locked in) -- **channel\_id** (hash, optional): The full channel\_id (funding txid Xored with output number) (always 64 characters) +- **channel\_id** (hash, optional): The full channel\_id (funding txid Xored with output number) - **funding\_txid** (txid, optional): ID of the funding transaction - **funding\_outnum** (u32, optional): The 0-based output number of the funding transaction which opens the channel - **initial\_feerate** (string, optional): For inflight opens, the first feerate used to initiate the channel open @@ -103,7 +103,7 @@ On success, an object containing **channels** is returned. It is an array of ob - **id** (u64): Unique ID for this htlc on this channel in this direction - **amount\_msat** (msat): Amount send/received for this HTLC - **expiry** (u32): Block this HTLC expires at (after which an `in` direction HTLC will be returned to the peer, an `out` returned to us). If this expiry is too close, lightningd(8) will automatically unilaterally close the channel in order to enforce the timeout onchain. - - **payment\_hash** (hash): the hash of the payment\_preimage which will prove payment (always 64 characters) + - **payment\_hash** (hash): the hash of the payment\_preimage which will prove payment - **local\_trimmed** (boolean, optional): If this is too small to enforce onchain; it doesn't appear in the commitment transaction and will not be enforced in a unilateral close. Generally true if the HTLC (after subtracting onchain fees) is below the `dust_limit_msat` for the channel. (always *true*) - **status** (string, optional): set if this HTLC is currently waiting on a hook (and shows what plugin) @@ -191,4 +191,4 @@ Main web site: Lightning RFC site (BOLT \#9): -[comment]: # ( SHA256STAMP:64f0c002713473b6b8e06fc6b5eba0dc830b98df21f8aa5be9a80052c5b7a7e7) +[comment]: # ( SHA256STAMP:4abe4d7c2e43629d536f4ef906c579a36e2636b183692ffee1474a18438ab630) diff --git a/doc/lightning-listsendpays.7.md b/doc/lightning-listsendpays.7.md index 3070683ff2e3..53feb23927f0 100644 --- a/doc/lightning-listsendpays.7.md +++ b/doc/lightning-listsendpays.7.md @@ -27,7 +27,7 @@ On success, an object containing **payments** is returned. It is an array of ob - **id** (u64): unique ID for this payment attempt - **groupid** (u64): Grouping key to disambiguate multiple attempts to pay an invoice or the same payment\_hash -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): status of the payment (one of "pending", "failed", "complete") - **created\_at** (u64): the UNIX timestamp showing when this payment was initiated - **amount\_sent\_msat** (msat): The amount sent @@ -41,7 +41,7 @@ On success, an object containing **payments** is returned. It is an array of ob If **status** is "complete": - - **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** (always 64 characters) + - **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** If **status** is "failed": @@ -65,4 +65,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:c00443620140062eadf4ccdc32f08102d8278c80975437a708e1b4913b6ab85a) +[comment]: # ( SHA256STAMP:9de82a239bae09bf777bdb988170c7ec43946ea49c9dfa908430f65d0a42fdbb) diff --git a/doc/lightning-makesecret.7.md b/doc/lightning-makesecret.7.md index bec1269a5b95..3faa4ab78040 100644 --- a/doc/lightning-makesecret.7.md +++ b/doc/lightning-makesecret.7.md @@ -20,7 +20,7 @@ RETURN VALUE [comment]: # (GENERATE-FROM-SCHEMA-START) On success, an object is returned, containing: -- **secret** (secret): the pseudorandom key derived from HSM\_secret (always 64 characters) +- **secret** (secret): the pseudorandom key derived from HSM\_secret [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -38,4 +38,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:3c8977e48a221ea5354f1affb1ae86b606aca2df4af32bad0a101386a12556a3) +[comment]: # ( SHA256STAMP:94f3d533a330909b8f46d03d6a3fdd4c54105a948ee7ffa23ed853d785dd4f60) diff --git a/doc/lightning-pay.7.md b/doc/lightning-pay.7.md index 280bae16dd4e..4ef5d9788065 100644 --- a/doc/lightning-pay.7.md +++ b/doc/lightning-pay.7.md @@ -96,8 +96,8 @@ RETURN VALUE [comment]: # (GENERATE-FROM-SCHEMA-START) On success, an object is returned, containing: -- **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** (always 64 characters) -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **created\_at** (number): the UNIX timestamp showing when this payment was initiated - **parts** (u32): how many attempts this took - **amount\_msat** (msat): Amount the recipient received @@ -168,4 +168,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:6e644d1071a3e0b9f53e67ced3b29702d93849402e3e6893f41db9f4b23da064) +[comment]: # ( SHA256STAMP:d220052ef3c013560eb3b4b379a5f5aa5ff4ce719b0bd2f05f0645cfc25804f9) diff --git a/doc/lightning-sendonion.7.md b/doc/lightning-sendonion.7.md index 051edc79e8c1..9242a01de389 100644 --- a/doc/lightning-sendonion.7.md +++ b/doc/lightning-sendonion.7.md @@ -94,7 +94,7 @@ RETURN VALUE On success, an object is returned, containing: - **id** (u64): unique ID for this payment attempt -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): status of the payment (could be complete if already sent previously) (one of "pending", "complete") - **created\_at** (u64): the UNIX timestamp showing when this payment was initiated - **amount\_sent\_msat** (msat): The amount sent @@ -107,7 +107,7 @@ On success, an object is returned, containing: If **status** is "complete": - - **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** (always 64 characters) + - **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** If **status** is "pending": @@ -135,4 +135,4 @@ RESOURCES Main web site: [bolt04]: https://github.com/lightning/bolts/blob/master/04-onion-routing.md -[comment]: # ( SHA256STAMP:88fc091802bfa0295ce3b2e445160d2357a66d6501328bdb086498960b2f915d) +[comment]: # ( SHA256STAMP:ffc19dc92199d296f1f8bc974923abd76378f361ff68697137b9dab864d65094) diff --git a/doc/lightning-sendpay.7.md b/doc/lightning-sendpay.7.md index bb2271b92938..7051e0b563b3 100644 --- a/doc/lightning-sendpay.7.md +++ b/doc/lightning-sendpay.7.md @@ -68,7 +68,7 @@ RETURN VALUE On success, an object is returned, containing: - **id** (u64): unique ID for this payment attempt -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): status of the payment (could be complete if already sent previously) (one of "pending", "complete") - **created\_at** (u64): the UNIX timestamp showing when this payment was initiated - **amount\_sent\_msat** (msat): The amount sent @@ -83,7 +83,7 @@ On success, an object is returned, containing: If **status** is "complete": - - **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** (always 64 characters) + - **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** If **status** is "pending": @@ -142,4 +142,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:50657fd6d5bef4f88ae1d38c5ddfe0d42eb67f222d51e90b03354c68c56f7905) +[comment]: # ( SHA256STAMP:1cc526944ea1119f507383f58a9c251dff2ca0b86c15675317753328549be78d) diff --git a/doc/lightning-waitanyinvoice.7.md b/doc/lightning-waitanyinvoice.7.md index c48d4fbc4ce4..850a0111c730 100644 --- a/doc/lightning-waitanyinvoice.7.md +++ b/doc/lightning-waitanyinvoice.7.md @@ -38,7 +38,7 @@ On success, an object is returned, containing: - **label** (string): unique label supplied at invoice creation - **description** (string): description used in the invoice -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): Whether it's paid or expired (one of "paid", "expired") - **expires\_at** (u64): UNIX timestamp of when it will become / became unpayable - **amount\_msat** (msat, optional): the amount required to pay this invoice @@ -50,7 +50,7 @@ If **status** is "paid": - **pay\_index** (u64): Unique incrementing index for this payment - **amount\_received\_msat** (msat): the amount actually received (could be slightly greater than *amount\_msat*, since clients may overpay) - **paid\_at** (u64): UNIX timestamp of when it was paid - - **payment\_preimage** (secret): proof of payment (always 64 characters) + - **payment\_preimage** (secret): proof of payment [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -75,4 +75,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:846510edabc52b21c0ae6482a49e373ebe7feeb4697b6a6f48d85b30351086f2) +[comment]: # ( SHA256STAMP:455c57c49af09b9360b8c4a052828ce71f75cb178a63646eb2621e9e9f0faa5a) diff --git a/doc/lightning-waitinvoice.7.md b/doc/lightning-waitinvoice.7.md index 254c24fb4aca..f2a8767949bb 100644 --- a/doc/lightning-waitinvoice.7.md +++ b/doc/lightning-waitinvoice.7.md @@ -20,7 +20,7 @@ On success, an object is returned, containing: - **label** (string): unique label supplied at invoice creation - **description** (string): description used in the invoice -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): Whether it's paid or expired (one of "paid", "expired") - **expires\_at** (u64): UNIX timestamp of when it will become / became unpayable - **amount\_msat** (msat, optional): the amount required to pay this invoice @@ -32,7 +32,7 @@ If **status** is "paid": - **pay\_index** (u64): Unique incrementing index for this payment - **amount\_received\_msat** (msat): the amount actually received (could be slightly greater than *amount\_msat*, since clients may overpay) - **paid\_at** (u64): UNIX timestamp of when it was paid - - **payment\_preimage** (secret): proof of payment (always 64 characters) + - **payment\_preimage** (secret): proof of payment [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -60,4 +60,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:846510edabc52b21c0ae6482a49e373ebe7feeb4697b6a6f48d85b30351086f2) +[comment]: # ( SHA256STAMP:455c57c49af09b9360b8c4a052828ce71f75cb178a63646eb2621e9e9f0faa5a) diff --git a/doc/lightning-waitsendpay.7.md b/doc/lightning-waitsendpay.7.md index f441be795ad3..61f6de122be4 100644 --- a/doc/lightning-waitsendpay.7.md +++ b/doc/lightning-waitsendpay.7.md @@ -36,7 +36,7 @@ RETURN VALUE On success, an object is returned, containing: - **id** (u64): unique ID for this payment attempt -- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): status of the payment (always "complete") - **created\_at** (u64): the UNIX timestamp showing when this payment was initiated - **amount\_sent\_msat** (msat): The amount sent @@ -51,7 +51,7 @@ On success, an object is returned, containing: If **status** is "complete": - - **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** (always 64 characters) + - **payment\_preimage** (secret): the proof of payment: SHA256 of this **payment\_hash** [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -104,4 +104,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:6131f1442571e836e38b1570e75639c3d499b4602659a19ad03aa7156d58d949) +[comment]: # ( SHA256STAMP:42da651674955a4e839abc7677263fbdda3a62ba3ce7c251b698828d604b0d4c) diff --git a/doc/schemas/createinvoice.schema.json b/doc/schemas/createinvoice.schema.json index e25ce1232f5c..0a941b3fcce1 100644 --- a/doc/schemas/createinvoice.schema.json +++ b/doc/schemas/createinvoice.schema.json @@ -24,9 +24,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "amount_msat": { "type": "msat", @@ -63,9 +61,7 @@ }, "payment_preimage": { "type": "secret", - "description": "the proof of payment: SHA256 of this **payment_hash**", - "maxLength": 64, - "minLength": 64 + "description": "the proof of payment: SHA256 of this **payment_hash**" }, "local_offer_id": { "type": "hex", diff --git a/doc/schemas/createonion.schema.json b/doc/schemas/createonion.schema.json index eab28f2a3c0f..852e83e36a88 100644 --- a/doc/schemas/createonion.schema.json +++ b/doc/schemas/createonion.schema.json @@ -16,9 +16,7 @@ "description": "one shared secret for each node in the *hops* parameter", "items": { "type": "secret", - "description": "the shared secret with this hop", - "maxLength": 64, - "minLength": 64 + "description": "the shared secret with this hop" } } } diff --git a/doc/schemas/delinvoice.schema.json b/doc/schemas/delinvoice.schema.json index e48ce5554e4d..92d7eac267f5 100644 --- a/doc/schemas/delinvoice.schema.json +++ b/doc/schemas/delinvoice.schema.json @@ -34,9 +34,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -155,9 +153,7 @@ }, "payment_preimage": { "type": "secret", - "description": "SHA256 of this is the *payment_hash* offered in the invoice", - "maxLength": 64, - "minLength": 64 + "description": "SHA256 of this is the *payment_hash* offered in the invoice" } } }, diff --git a/doc/schemas/invoice.schema.json b/doc/schemas/invoice.schema.json index 8092b576da3a..292e92072821 100644 --- a/doc/schemas/invoice.schema.json +++ b/doc/schemas/invoice.schema.json @@ -15,15 +15,11 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "payment_secret": { "type": "secret", - "description": "the *payment_secret* to place in the onion", - "maxLength": 64, - "minLength": 64 + "description": "the *payment_secret* to place in the onion" }, "expires_at": { "type": "u64", diff --git a/doc/schemas/keysend.schema.json b/doc/schemas/keysend.schema.json index 1881260cdb08..560d5dc58652 100644 --- a/doc/schemas/keysend.schema.json +++ b/doc/schemas/keysend.schema.json @@ -14,9 +14,7 @@ "properties": { "payment_preimage": { "type": "secret", - "description": "the proof of payment: SHA256 of this **payment_hash**", - "maxLength": 64, - "minLength": 64 + "description": "the proof of payment: SHA256 of this **payment_hash**" }, "destination": { "type": "pubkey", @@ -24,9 +22,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "created_at": { "type": "number", diff --git a/doc/schemas/listinvoices.schema.json b/doc/schemas/listinvoices.schema.json index 9fef65e06767..9c0172916869 100644 --- a/doc/schemas/listinvoices.schema.json +++ b/doc/schemas/listinvoices.schema.json @@ -28,9 +28,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -120,9 +118,7 @@ }, "payment_preimage": { "type": "secret", - "description": "proof of payment", - "maxLength": 64, - "minLength": 64 + "description": "proof of payment" } } }, diff --git a/doc/schemas/listpeerchannels.schema.json b/doc/schemas/listpeerchannels.schema.json index 97e58c18b0e9..7764716bf5dc 100644 --- a/doc/schemas/listpeerchannels.schema.json +++ b/doc/schemas/listpeerchannels.schema.json @@ -77,9 +77,7 @@ }, "channel_id": { "type": "hash", - "description": "The full channel_id (funding txid Xored with output number)", - "minLength": 64, - "maxLength": 64 + "description": "The full channel_id (funding txid Xored with output number)" }, "funding_txid": { "type": "txid", @@ -502,9 +500,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the payment_preimage which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the payment_preimage which will prove payment" }, "local_trimmed": { "type": "boolean", diff --git a/doc/schemas/listsendpays.schema.json b/doc/schemas/listsendpays.schema.json index ed0cb013ed99..65f56daf3cf1 100644 --- a/doc/schemas/listsendpays.schema.json +++ b/doc/schemas/listsendpays.schema.json @@ -34,9 +34,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -122,9 +120,7 @@ "bolt12": {}, "payment_preimage": { "type": "secret", - "description": "the proof of payment: SHA256 of this **payment_hash**", - "maxLength": 64, - "minLength": 64 + "description": "the proof of payment: SHA256 of this **payment_hash**" } } } diff --git a/doc/schemas/makesecret.schema.json b/doc/schemas/makesecret.schema.json index ce17c2fcb7cd..b9b8ff50808c 100644 --- a/doc/schemas/makesecret.schema.json +++ b/doc/schemas/makesecret.schema.json @@ -8,9 +8,7 @@ "properties": { "secret": { "type": "secret", - "description": "the pseudorandom key derived from HSM_secret", - "maxLength": 64, - "minLength": 64 + "description": "the pseudorandom key derived from HSM_secret" } } } diff --git a/doc/schemas/pay.schema.json b/doc/schemas/pay.schema.json index dc2965fab9c8..13d7d7612e22 100644 --- a/doc/schemas/pay.schema.json +++ b/doc/schemas/pay.schema.json @@ -14,9 +14,7 @@ "properties": { "payment_preimage": { "type": "secret", - "description": "the proof of payment: SHA256 of this **payment_hash**", - "maxLength": 64, - "minLength": 64 + "description": "the proof of payment: SHA256 of this **payment_hash**" }, "destination": { "type": "pubkey", @@ -24,9 +22,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "created_at": { "type": "number", diff --git a/doc/schemas/sendonion.schema.json b/doc/schemas/sendonion.schema.json index 5b39cf772313..8a49a5c19ad2 100644 --- a/doc/schemas/sendonion.schema.json +++ b/doc/schemas/sendonion.schema.json @@ -16,9 +16,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -101,9 +99,7 @@ "partid": {}, "payment_preimage": { "type": "secret", - "description": "the proof of payment: SHA256 of this **payment_hash**", - "maxLength": 64, - "minLength": 64 + "description": "the proof of payment: SHA256 of this **payment_hash**" } } } diff --git a/doc/schemas/sendpay.schema.json b/doc/schemas/sendpay.schema.json index 623fc66d4ef0..5cda57faea66 100644 --- a/doc/schemas/sendpay.schema.json +++ b/doc/schemas/sendpay.schema.json @@ -20,9 +20,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -110,9 +108,7 @@ "bolt12": {}, "payment_preimage": { "type": "secret", - "description": "the proof of payment: SHA256 of this **payment_hash**", - "maxLength": 64, - "minLength": 64 + "description": "the proof of payment: SHA256 of this **payment_hash**" } } } diff --git a/doc/schemas/waitanyinvoice.schema.json b/doc/schemas/waitanyinvoice.schema.json index 7a5b26f45a31..1a1802a18074 100644 --- a/doc/schemas/waitanyinvoice.schema.json +++ b/doc/schemas/waitanyinvoice.schema.json @@ -20,9 +20,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -99,9 +97,7 @@ }, "payment_preimage": { "type": "secret", - "description": "proof of payment", - "maxLength": 64, - "minLength": 64 + "description": "proof of payment" } } }, diff --git a/doc/schemas/waitinvoice.schema.json b/doc/schemas/waitinvoice.schema.json index 7a5b26f45a31..1a1802a18074 100644 --- a/doc/schemas/waitinvoice.schema.json +++ b/doc/schemas/waitinvoice.schema.json @@ -20,9 +20,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -99,9 +97,7 @@ }, "payment_preimage": { "type": "secret", - "description": "proof of payment", - "maxLength": 64, - "minLength": 64 + "description": "proof of payment" } } }, diff --git a/doc/schemas/waitsendpay.schema.json b/doc/schemas/waitsendpay.schema.json index afb9b8af4363..3b73bec93388 100644 --- a/doc/schemas/waitsendpay.schema.json +++ b/doc/schemas/waitsendpay.schema.json @@ -20,9 +20,7 @@ }, "payment_hash": { "type": "hash", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -109,9 +107,7 @@ "bolt12": {}, "payment_preimage": { "type": "secret", - "description": "the proof of payment: SHA256 of this **payment_hash**", - "maxLength": 64, - "minLength": 64 + "description": "the proof of payment: SHA256 of this **payment_hash**" } } } From f1efb6589815e1bc393f340475f85182f50a1c5e Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:17 +1030 Subject: [PATCH 19/36] doc: use specific types in schema rather than "hex". We have "secret" and "hash" types which are often more appropriate. Signed-off-by: Rusty Russell --- cln-grpc/src/convert.rs | 6 +++--- cln-rpc/src/model.rs | 6 +++--- doc/lightning-decode.7.md | 12 ++++++------ doc/lightning-decodepay.7.md | 8 ++++---- doc/lightning-delpay.7.md | 6 +++--- doc/lightning-disableoffer.7.md | 4 ++-- doc/lightning-listinvoices.7.md | 4 ++-- doc/lightning-listoffers.7.md | 4 ++-- doc/lightning-listpays.7.md | 6 +++--- doc/lightning-offer.7.md | 4 ++-- doc/lightning-sendinvoice.7.md | 6 +++--- doc/schemas/decode.schema.json | 24 ++++++++---------------- doc/schemas/decodepay.schema.json | 18 ++++++------------ doc/schemas/delpay.request.json | 6 ++---- doc/schemas/delpay.schema.json | 12 ++++-------- doc/schemas/disableoffer.schema.json | 6 ++---- doc/schemas/listinvoices.schema.json | 6 ++---- doc/schemas/listoffers.schema.json | 6 ++---- doc/schemas/listpays.schema.json | 12 ++++-------- doc/schemas/offer.schema.json | 6 ++---- doc/schemas/sendinvoice.schema.json | 12 ++++-------- 21 files changed, 69 insertions(+), 105 deletions(-) diff --git a/cln-grpc/src/convert.rs b/cln-grpc/src/convert.rs index 6b8032f5ad2d..a77a3311a35a 100644 --- a/cln-grpc/src/convert.rs +++ b/cln-grpc/src/convert.rs @@ -528,7 +528,7 @@ impl From for pb::ListinvoicesInvoices { amount_msat: c.amount_msat.map(|f| f.into()), // Rule #2 for type msat? bolt11: c.bolt11, // Rule #2 for type string? bolt12: c.bolt12, // Rule #2 for type string? - local_offer_id: c.local_offer_id.map(|v| hex::decode(v).unwrap()), // Rule #2 for type hex? + local_offer_id: c.local_offer_id.map(|v| v.to_vec()), // Rule #2 for type hash? invreq_payer_note: c.invreq_payer_note, // Rule #2 for type string? pay_index: c.pay_index, // Rule #2 for type u64? amount_received_msat: c.amount_received_msat.map(|f| f.into()), // Rule #2 for type msat? @@ -1041,7 +1041,7 @@ impl From for pb::ListforwardsResponse { impl From for pb::ListpaysPays { fn from(c: responses::ListpaysPays) -> Self { Self { - payment_hash: hex::decode(&c.payment_hash).unwrap(), // Rule #2 for type hex + payment_hash: c.payment_hash.to_vec(), // Rule #2 for type hash status: c.status as i32, destination: c.destination.map(|v| v.serialize().to_vec()), // Rule #2 for type pubkey? created_at: c.created_at, // Rule #2 for type u64 @@ -1050,7 +1050,7 @@ impl From for pb::ListpaysPays { bolt11: c.bolt11, // Rule #2 for type string? description: c.description, // Rule #2 for type string? bolt12: c.bolt12, // Rule #2 for type string? - preimage: c.preimage.map(|v| hex::decode(v).unwrap()), // Rule #2 for type hex? + preimage: c.preimage.map(|v| v.to_vec()), // Rule #2 for type secret? number_of_parts: c.number_of_parts, // Rule #2 for type u64? erroronion: c.erroronion.map(|v| hex::decode(v).unwrap()), // Rule #2 for type hex? } diff --git a/cln-rpc/src/model.rs b/cln-rpc/src/model.rs index cca63253dafc..1c35547413ce 100644 --- a/cln-rpc/src/model.rs +++ b/cln-rpc/src/model.rs @@ -2362,7 +2362,7 @@ pub mod responses { #[serde(skip_serializing_if = "Option::is_none")] pub bolt12: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub local_offer_id: Option, + pub local_offer_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub invreq_payer_note: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -3430,7 +3430,7 @@ pub mod responses { } #[derive(Clone, Debug, Deserialize, Serialize)] pub struct ListpaysPays { - pub payment_hash: String, + pub payment_hash: Sha256, // Path `ListPays.pays[].status` pub status: ListpaysPaysStatus, #[serde(skip_serializing_if = "Option::is_none")] @@ -3447,7 +3447,7 @@ pub mod responses { #[serde(skip_serializing_if = "Option::is_none")] pub bolt12: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub preimage: Option, + pub preimage: Option, #[serde(skip_serializing_if = "Option::is_none")] pub number_of_parts: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/doc/lightning-decode.7.md b/doc/lightning-decode.7.md index 2c4bfc37e122..3ae556d4e6fe 100644 --- a/doc/lightning-decode.7.md +++ b/doc/lightning-decode.7.md @@ -32,8 +32,8 @@ If **type** is "bolt12 offer", and **valid** is *true*: - **offer\_id** (hex): the id we use to identify this offer (always 64 characters) - **offer\_description** (string): the description of the purpose of the offer - **offer\_node\_id** (pubkey): public key of the offering node - - **offer\_chains** (array of hexs, optional): which blockchains this offer is for (missing implies bitcoin mainnet only): - - the genesis blockhash (always 64 characters) + - **offer\_chains** (array of hashs, optional): which blockchains this offer is for (missing implies bitcoin mainnet only): + - the genesis blockhash - **offer\_metadata** (hex, optional): any metadata the creater of the offer includes - **offer\_currency** (string, optional): ISO 4217 code of the currency (missing implies Bitcoin) (always 3 characters) - **currency\_minor\_unit** (u32, optional): the number of decimal places to apply to amount (if currency known) @@ -239,13 +239,13 @@ If **type** is "bolt11 invoice", and **valid** is *true*: - **created\_at** (u64): the UNIX-style timestamp of the invoice - **expiry** (u64): the number of seconds this is valid after `created_at` - **payee** (pubkey): the public key of the recipient - - **payment\_hash** (hex): the hash of the *payment\_preimage* (always 64 characters) + - **payment\_hash** (hash): the hash of the *payment\_preimage* - **signature** (signature): signature of the *payee* on this invoice - **min\_final\_cltv\_expiry** (u32): the minimum CLTV delay for the final node - **amount\_msat** (msat, optional): Amount the invoice asked for - **description** (string, optional): the description of the purpose of the purchase - - **description\_hash** (hex, optional): the hash of the description, in place of *description* (always 64 characters) - - **payment\_secret** (hex, optional): the secret to hand to the payee node (always 64 characters) + - **description\_hash** (hash, optional): the hash of the description, in place of *description* + - **payment\_secret** (secret, optional): the secret to hand to the payee node - **features** (hex, optional): the features bitmap for this invoice - **payment\_metadata** (hex, optional): the payment\_metadata to put in the payment - **fallbacks** (array of objects, optional): onchain addresses: @@ -303,4 +303,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:25247da6ea63f7fa229b7d25afd8cc2558ded16219c4a49ca3567dbb522f0153) +[comment]: # ( SHA256STAMP:ba144aad8e34d9a8581161be01fa9a5e0107d068ad35411a278539503446768b) diff --git a/doc/lightning-decodepay.7.md b/doc/lightning-decodepay.7.md index cb256b184753..f7d6f5d7c77c 100644 --- a/doc/lightning-decodepay.7.md +++ b/doc/lightning-decodepay.7.md @@ -22,13 +22,13 @@ On success, an object is returned, containing: - **created\_at** (u64): the UNIX-style timestamp of the invoice - **expiry** (u64): the number of seconds this is valid after *timestamp* - **payee** (pubkey): the public key of the recipient -- **payment\_hash** (hex): the hash of the *payment\_preimage* (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* - **signature** (signature): signature of the *payee* on this invoice - **min\_final\_cltv\_expiry** (u32): the minimum CLTV delay for the final node - **amount\_msat** (msat, optional): Amount the invoice asked for - **description** (string, optional): the description of the purpose of the purchase -- **description\_hash** (hex, optional): the hash of the description, in place of *description* (always 64 characters) -- **payment\_secret** (hex, optional): the secret to hand to the payee node (always 64 characters) +- **description\_hash** (hash, optional): the hash of the description, in place of *description* +- **payment\_secret** (hash, optional): the secret to hand to the payee node - **features** (hex, optional): the features bitmap for this invoice - **payment\_metadata** (hex, optional): the payment\_metadata to put in the payment - **fallbacks** (array of objects, optional): onchain addresses: @@ -71,4 +71,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:1d4a7f4577ffa26f34da9ea3a0ed2f7da26f86a4ce554b8bf20785a3de20fdfe) +[comment]: # ( SHA256STAMP:d287a96b5495b4be07d8a20633b9a6d5179ef74fc33b1b517c1b201e1b86e9aa) diff --git a/doc/lightning-delpay.7.md b/doc/lightning-delpay.7.md index 1bc7d3ef954b..810b480121d5 100644 --- a/doc/lightning-delpay.7.md +++ b/doc/lightning-delpay.7.md @@ -42,7 +42,7 @@ payments will be returned -- one payment object for each partid. On success, an object containing **payments** is returned. It is an array of objects, where each object contains: - **id** (u64): unique ID for this payment attempt -- **payment\_hash** (hex): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): status of the payment (one of "pending", "failed", "complete") - **amount\_sent\_msat** (msat): the amount we actually sent, including fees - **created\_at** (u64): the UNIX timestamp showing when this payment was initiated @@ -51,7 +51,7 @@ On success, an object containing **payments** is returned. It is an array of ob - **amount\_msat** (msat, optional): the amount the destination received, if known - **completed\_at** (u64, optional): the UNIX timestamp showing when this payment was completed - **groupid** (u64, optional): Grouping key to disambiguate multiple attempts to pay an invoice or the same payment\_hash -- **payment\_preimage** (hex, optional): proof of payment (always 64 characters) +- **payment\_preimage** (secret, optional): proof of payment - **label** (string, optional): the label, if given to sendpay - **bolt11** (string, optional): the bolt11 string (if pay supplied one) - **bolt12** (string, optional): the bolt12 string (if supplied for pay: **experimental-offers** only). @@ -107,4 +107,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:c6d248396e04a0ef3506dbd5319231cf8d4cb2522a4e039d5e3622aed5ab4496) +[comment]: # ( SHA256STAMP:04fdf8931ea040a3433df9e25b1db1e808e733ad3a5b2586f6edd030ae6f165a) diff --git a/doc/lightning-disableoffer.7.md b/doc/lightning-disableoffer.7.md index 92ccddcc01a6..2ba1794a49ba 100644 --- a/doc/lightning-disableoffer.7.md +++ b/doc/lightning-disableoffer.7.md @@ -36,7 +36,7 @@ Note: the returned object is the same format as **listoffers**. [comment]: # (GENERATE-FROM-SCHEMA-START) On success, an object is returned, containing: -- **offer\_id** (hex): the merkle hash of the offer (always 64 characters) +- **offer\_id** (hash): the merkle hash of the offer - **active** (boolean): Whether the offer can produce invoices/payments (always *false*) - **single\_use** (boolean): Whether the offer is disabled after first successful use - **bolt12** (string): The bolt12 string representing this offer @@ -74,4 +74,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:339f2e5a5a587414ba6ad3e312c6cff5525bae86addb76bb5656665eeeef3dd6) +[comment]: # ( SHA256STAMP:e03f739fb57f18205421785604ea542e931db42b1accfcb196dfc147a7c8bf75) diff --git a/doc/lightning-listinvoices.7.md b/doc/lightning-listinvoices.7.md index a2d4e41eacc8..4e1de86e457b 100644 --- a/doc/lightning-listinvoices.7.md +++ b/doc/lightning-listinvoices.7.md @@ -31,7 +31,7 @@ On success, an object containing **invoices** is returned. It is an array of ob - **amount\_msat** (msat, optional): the amount required to pay this invoice - **bolt11** (string, optional): the BOLT11 string (always present unless *bolt12* is) - **bolt12** (string, optional): the BOLT12 string (always present unless *bolt11* is) -- **local\_offer\_id** (hex, optional): the *id* of our offer which created this invoice (**experimental-offers** only). (always 64 characters) +- **local\_offer\_id** (hash, optional): the *id* of our offer which created this invoice (**experimental-offers** only). - **invreq\_payer\_note** (string, optional): the optional *invreq\_payer\_note* from invoice\_request which created this invoice (**experimental-offers** only). If **status** is "paid": @@ -58,4 +58,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:258d690221dce5a8811f361e59a5c0059190f140e420c65cd37cfd0987efea3a) +[comment]: # ( SHA256STAMP:1b31938109207decabf1e0f25cc9607dc03de7f043f4e5fcfbfb8c85ffacec8c) diff --git a/doc/lightning-listoffers.7.md b/doc/lightning-listoffers.7.md index fe6c8ea68430..6a809807df6c 100644 --- a/doc/lightning-listoffers.7.md +++ b/doc/lightning-listoffers.7.md @@ -32,7 +32,7 @@ RETURN VALUE [comment]: # (GENERATE-FROM-SCHEMA-START) On success, an object containing **offers** is returned. It is an array of objects, where each object contains: -- **offer\_id** (hex): the id of this offer (merkle hash of non-signature fields) (always 64 characters) +- **offer\_id** (hash): the id of this offer (merkle hash of non-signature fields) - **active** (boolean): whether this can still be used - **single\_use** (boolean): whether this expires as soon as it's paid - **bolt12** (string): the bolt12 encoding of the offer @@ -81,4 +81,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:b968ff5dd2eb57cb030c7a9fb73d3d305e69f23dc2ccd599573fb345e0f58385) +[comment]: # ( SHA256STAMP:863d9f666cbbbd013b86b4075a7c8b7e7bda47049c562cba080d0a88626636a1) diff --git a/doc/lightning-listpays.7.md b/doc/lightning-listpays.7.md index 44dfa66fb44e..2ef4ffa5244c 100644 --- a/doc/lightning-listpays.7.md +++ b/doc/lightning-listpays.7.md @@ -19,7 +19,7 @@ RETURN VALUE [comment]: # (GENERATE-FROM-SCHEMA-START) On success, an object containing **pays** is returned. It is an array of objects, where each object contains: -- **payment\_hash** (hex): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): status of the payment (one of "pending", "failed", "complete") - **created\_at** (u64): the UNIX timestamp showing when this payment was initiated - **destination** (pubkey, optional): the final destination of the payment if known @@ -31,7 +31,7 @@ On success, an object containing **pays** is returned. It is an array of object If **status** is "complete": - - **preimage** (hex): proof of payment (always 64 characters) + - **preimage** (secret): proof of payment - **number\_of\_parts** (u64, optional): the number of parts for a successful payment (only if more than one). If **status** is "failed": @@ -57,4 +57,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:adf164794675b251cbef10f5c40dd2e05812fc681641e0631126bae2a8bc8883) +[comment]: # ( SHA256STAMP:716bcbf01d946c6e4da0bd2f6817c34e6471a1fcd2f0f388ce47984271285c72) diff --git a/doc/lightning-offer.7.md b/doc/lightning-offer.7.md index 24d7fa36f365..defe088fc8b6 100644 --- a/doc/lightning-offer.7.md +++ b/doc/lightning-offer.7.md @@ -94,7 +94,7 @@ RETURN VALUE [comment]: # (GENERATE-FROM-SCHEMA-START) On success, an object is returned, containing: -- **offer\_id** (hex): the id of this offer (merkle hash of non-signature fields) (always 64 characters) +- **offer\_id** (hash): the id of this offer (merkle hash of non-signature fields) - **active** (boolean): whether this can still be used (always *true*) - **single\_use** (boolean): whether this expires as soon as it's paid (reflects the *single\_use* parameter) - **bolt12** (string): the bolt12 encoding of the offer @@ -131,4 +131,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:46e6bfcfe48f709e2f2068250e28b53bedafb782db42791e722d301515fce070) +[comment]: # ( SHA256STAMP:3ad09aed48fb17db5fae6d401f21e50a4479e970199bd039b453868057829653) diff --git a/doc/lightning-sendinvoice.7.md b/doc/lightning-sendinvoice.7.md index 69610cb57507..d4a9c424396a 100644 --- a/doc/lightning-sendinvoice.7.md +++ b/doc/lightning-sendinvoice.7.md @@ -43,7 +43,7 @@ On success, an object is returned, containing: - **label** (string): unique label supplied at invoice creation - **description** (string): description used in the invoice -- **payment\_hash** (hex): the hash of the *payment\_preimage* which will prove payment (always 64 characters) +- **payment\_hash** (hash): the hash of the *payment\_preimage* which will prove payment - **status** (string): Whether it's paid, unpaid or unpayable (one of "unpaid", "paid", "expired") - **expires\_at** (u64): UNIX timestamp of when it will become / became unpayable - **amount\_msat** (msat, optional): the amount required to pay this invoice @@ -54,7 +54,7 @@ If **status** is "paid": - **pay\_index** (u64): Unique incrementing index for this payment - **amount\_received\_msat** (msat): the amount actually received (could be slightly greater than *amount\_msat*, since clients may overpay) - **paid\_at** (u64): UNIX timestamp of when it was paid - - **payment\_preimage** (hex): proof of payment (always 64 characters) + - **payment\_preimage** (secret): proof of payment [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -80,4 +80,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:a088e5202cb822518a2214b485083edf0a929290b17beb9a3930dd93d97411d0) +[comment]: # ( SHA256STAMP:7646a92936e1e79dbefe9cacf418ee15f148db467d780a9b39b90d46ca522539) diff --git a/doc/schemas/decode.schema.json b/doc/schemas/decode.schema.json index 9635b5693bad..39c1b361cd10 100644 --- a/doc/schemas/decode.schema.json +++ b/doc/schemas/decode.schema.json @@ -60,10 +60,8 @@ "type": "array", "description": "which blockchains this offer is for (missing implies bitcoin mainnet only)", "items": { - "type": "hex", - "description": "the genesis blockhash", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the genesis blockhash" } }, "offer_metadata": { @@ -1270,10 +1268,8 @@ "description": "Amount the invoice asked for" }, "payment_hash": { - "type": "hex", - "description": "the hash of the *payment_preimage*", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the hash of the *payment_preimage*" }, "signature": { "type": "signature", @@ -1284,20 +1280,16 @@ "description": "the description of the purpose of the purchase" }, "description_hash": { - "type": "hex", - "description": "the hash of the description, in place of *description*", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the hash of the description, in place of *description*" }, "min_final_cltv_expiry": { "type": "u32", "description": "the minimum CLTV delay for the final node" }, "payment_secret": { - "type": "hex", - "description": "the secret to hand to the payee node", - "maxLength": 64, - "minLength": 64 + "type": "secret", + "description": "the secret to hand to the payee node" }, "features": { "type": "hex", diff --git a/doc/schemas/decodepay.schema.json b/doc/schemas/decodepay.schema.json index 09fa334da8d7..e1d536e2848d 100644 --- a/doc/schemas/decodepay.schema.json +++ b/doc/schemas/decodepay.schema.json @@ -37,10 +37,8 @@ "description": "Amount the invoice asked for" }, "payment_hash": { - "type": "hex", - "description": "the hash of the *payment_preimage*", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the hash of the *payment_preimage*" }, "signature": { "type": "signature", @@ -51,20 +49,16 @@ "description": "the description of the purpose of the purchase" }, "description_hash": { - "type": "hex", - "description": "the hash of the description, in place of *description*", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the hash of the description, in place of *description*" }, "min_final_cltv_expiry": { "type": "u32", "description": "the minimum CLTV delay for the final node" }, "payment_secret": { - "type": "hex", - "description": "the secret to hand to the payee node", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the secret to hand to the payee node" }, "features": { "type": "hex", diff --git a/doc/schemas/delpay.request.json b/doc/schemas/delpay.request.json index 0b341c3041ec..d670094e6388 100644 --- a/doc/schemas/delpay.request.json +++ b/doc/schemas/delpay.request.json @@ -8,10 +8,8 @@ "additionalProperties": false, "properties": { "payment_hash": { - "type": "hex", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", diff --git a/doc/schemas/delpay.schema.json b/doc/schemas/delpay.schema.json index fee03413c992..d1d6f81e77ce 100644 --- a/doc/schemas/delpay.schema.json +++ b/doc/schemas/delpay.schema.json @@ -24,10 +24,8 @@ "description": "unique ID for this payment attempt" }, "payment_hash": { - "type": "hex", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -73,10 +71,8 @@ "description": "Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash" }, "payment_preimage": { - "type": "hex", - "description": "proof of payment", - "maxLength": 64, - "minLength": 64 + "type": "secret", + "description": "proof of payment" }, "label": { "type": "string", diff --git a/doc/schemas/disableoffer.schema.json b/doc/schemas/disableoffer.schema.json index ccb64d27388d..3c4a28446882 100644 --- a/doc/schemas/disableoffer.schema.json +++ b/doc/schemas/disableoffer.schema.json @@ -11,10 +11,8 @@ "additionalProperties": false, "properties": { "offer_id": { - "type": "hex", - "description": "the merkle hash of the offer", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the merkle hash of the offer" }, "active": { "type": "boolean", diff --git a/doc/schemas/listinvoices.schema.json b/doc/schemas/listinvoices.schema.json index 9c0172916869..c18b48ef0b6e 100644 --- a/doc/schemas/listinvoices.schema.json +++ b/doc/schemas/listinvoices.schema.json @@ -59,10 +59,8 @@ "description": "the BOLT12 string (always present unless *bolt11* is)" }, "local_offer_id": { - "type": "hex", - "description": "the *id* of our offer which created this invoice (**experimental-offers** only).", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the *id* of our offer which created this invoice (**experimental-offers** only)." }, "invreq_payer_note": { "type": "string", diff --git a/doc/schemas/listoffers.schema.json b/doc/schemas/listoffers.schema.json index a97287c38ca3..340bb927d00f 100644 --- a/doc/schemas/listoffers.schema.json +++ b/doc/schemas/listoffers.schema.json @@ -20,10 +20,8 @@ ], "properties": { "offer_id": { - "type": "hex", - "description": "the id of this offer (merkle hash of non-signature fields)", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the id of this offer (merkle hash of non-signature fields)" }, "active": { "type": "boolean", diff --git a/doc/schemas/listpays.schema.json b/doc/schemas/listpays.schema.json index 9fc6cf83582a..9398d403a1c3 100644 --- a/doc/schemas/listpays.schema.json +++ b/doc/schemas/listpays.schema.json @@ -18,10 +18,8 @@ ], "properties": { "payment_hash": { - "type": "hex", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -92,10 +90,8 @@ "amount_msat": {}, "amount_sent_msat": {}, "preimage": { - "type": "hex", - "description": "proof of payment", - "maxLength": 64, - "minLength": 64 + "type": "secret", + "description": "proof of payment" }, "number_of_parts": { "type": "u64", diff --git a/doc/schemas/offer.schema.json b/doc/schemas/offer.schema.json index fc21a80d3c97..671de38dc900 100644 --- a/doc/schemas/offer.schema.json +++ b/doc/schemas/offer.schema.json @@ -12,10 +12,8 @@ ], "properties": { "offer_id": { - "type": "hex", - "description": "the id of this offer (merkle hash of non-signature fields)", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the id of this offer (merkle hash of non-signature fields)" }, "active": { "type": "boolean", diff --git a/doc/schemas/sendinvoice.schema.json b/doc/schemas/sendinvoice.schema.json index c274f20e33b5..bba697e445cf 100644 --- a/doc/schemas/sendinvoice.schema.json +++ b/doc/schemas/sendinvoice.schema.json @@ -19,10 +19,8 @@ "description": "description used in the invoice" }, "payment_hash": { - "type": "hex", - "description": "the hash of the *payment_preimage* which will prove payment", - "maxLength": 64, - "minLength": 64 + "type": "hash", + "description": "the hash of the *payment_preimage* which will prove payment" }, "status": { "type": "string", @@ -94,10 +92,8 @@ "description": "UNIX timestamp of when it was paid" }, "payment_preimage": { - "type": "hex", - "description": "proof of payment", - "maxLength": 64, - "minLength": 64 + "type": "secret", + "description": "proof of payment" } } } From c8a173b5bfc78eb6ea1cb9c892d6cb5e5d323ad9 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:17 +1030 Subject: [PATCH 20/36] plugins/sql: initial commit of new plugin. This is designed to allow you to perform complex server-side queries. Signed-off-by: Rusty Russell --- plugins/Makefile | 7 + plugins/sql.c | 556 +++++++++++++++++++++++++++++++++++++++++++ tests/test_plugin.py | 19 ++ 3 files changed, 582 insertions(+) create mode 100644 plugins/sql.c diff --git a/plugins/Makefile b/plugins/Makefile index ee35f1925397..ffbdd0deddc3 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -39,6 +39,10 @@ PLUGIN_FETCHINVOICE_SRC := plugins/fetchinvoice.c PLUGIN_FETCHINVOICE_OBJS := $(PLUGIN_FETCHINVOICE_SRC:.c=.o) PLUGIN_FETCHINVOICE_HEADER := +PLUGIN_SQL_SRC := plugins/sql.c +PLUGIN_SQL_HEADER := +PLUGIN_SQL_OBJS := $(PLUGIN_SQL_SRC:.c=.o) + PLUGIN_SPENDER_SRC := \ plugins/spender/fundchannel.c \ plugins/spender/main.c \ @@ -97,6 +101,7 @@ C_PLUGINS := \ plugins/offers \ plugins/pay \ plugins/txprepare \ + plugins/sql \ plugins/spenderp PLUGINS := $(C_PLUGINS) @@ -199,6 +204,8 @@ plugins/fetchinvoice: $(PLUGIN_FETCHINVOICE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_CO plugins/funder: bitcoin/psbt.o common/psbt_open.o $(PLUGIN_FUNDER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) +plugins/sql: $(PLUGIN_SQL_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) + # Generated from PLUGINS definition in plugins/Makefile ALL_C_HEADERS += plugins/list_of_builtin_plugins_gen.h plugins/list_of_builtin_plugins_gen.h: plugins/Makefile Makefile config.vars diff --git a/plugins/sql.c b/plugins/sql.c new file mode 100644 index 000000000000..155bac36e1f0 --- /dev/null +++ b/plugins/sql.c @@ -0,0 +1,556 @@ +/* Brilliant or insane? You decide! */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include + +/* TODO: + * 1. Generate from schemas. + * 2. Refresh time in API. + * 3. Colnames API to return dict. + * 4. sql-schemas command. + * 5. documentation. + * 6. test on mainnet. + * 7. Some cool query for documentation. + * 8. time_msec fields. + * 9. Primary key in schema? + * 10. Pagination API + */ +enum fieldtype { + /* Hex variants */ + FIELD_HEX, + FIELD_HASH, + FIELD_SECRET, + FIELD_PUBKEY, + FIELD_TXID, + /* Integer variants */ + FIELD_MSAT, + FIELD_INTEGER, + FIELD_U64, + FIELD_U32, + FIELD_U16, + FIELD_U8, + FIELD_BOOL, + /* Randoms */ + FIELD_NUMBER, + FIELD_STRING, + FIELD_SCID, +}; + +struct fieldtypemap { + const char *name; + const char *sqltype; +}; + +static const struct fieldtypemap fieldtypemap[] = { + { "hex", "BLOB" }, /* FIELD_HEX */ + { "hash", "BLOB" }, /* FIELD_HASH */ + { "secret", "BLOB" }, /* FIELD_SECRET */ + { "pubkey", "BLOB" }, /* FIELD_PUBKEY */ + { "txid", "BLOB" }, /* FIELD_TXID */ + { "msat", "INTEGER" }, /* FIELD_MSAT */ + { "integer", "INTEGER" }, /* FIELD_INTEGER */ + { "u64", "INTEGER" }, /* FIELD_U64 */ + { "u32", "INTEGER" }, /* FIELD_U32 */ + { "u16", "INTEGER" }, /* FIELD_U16 */ + { "u8", "INTEGER" }, /* FIELD_U8 */ + { "boolean", "INTEGER" }, /* FIELD_BOOL */ + { "number", "REAL" }, /* FIELD_NUMBER */ + { "string", "TEXT" }, /* FIELD_STRING */ + { "short_channel_id", "TEXT" }, /* FIELD_SCID */ +}; + +struct db_query { + sqlite3_stmt *stmt; + struct table_desc **tables; + const char *authfail; +}; + +struct table_desc { + /* e.g. peers for listpeers */ + const char *name; + const char **columns; + char *update_stmt; + enum fieldtype *fieldtypes; +}; +static STRMAP(struct table_desc *) tablemap; +static size_t max_dbmem = 500000000; +static struct sqlite3 *db; +static const char *dbfilename; + +static struct sqlite3 *sqlite_setup(struct plugin *plugin) +{ + int err; + struct sqlite3 *db; + char *errmsg; + + if (dbfilename) { + err = sqlite3_open_v2(dbfilename, &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + NULL); + } else { + err = sqlite3_open_v2("", &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + | SQLITE_OPEN_MEMORY, + NULL); + } + if (err != SQLITE_OK) + plugin_err(plugin, "Could not create db: errcode %u", err); + + sqlite3_extended_result_codes(db, 1); + + /* From https://www.sqlite.org/c3ref/set_authorizer.html: + * + * Applications that need to process SQL from untrusted + * sources might also consider lowering resource limits using + * sqlite3_limit() and limiting database size using the + * max_page_count PRAGMA in addition to using an authorizer. + */ + sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000); + sqlite3_limit(db, SQLITE_LIMIT_SQL_LENGTH, 10000); + sqlite3_limit(db, SQLITE_LIMIT_COLUMN, 100); + sqlite3_limit(db, SQLITE_LIMIT_EXPR_DEPTH, 100); + sqlite3_limit(db, SQLITE_LIMIT_COMPOUND_SELECT, 10); + sqlite3_limit(db, SQLITE_LIMIT_VDBE_OP, 1000); + sqlite3_limit(db, SQLITE_LIMIT_ATTACHED, 1); + sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 500); + sqlite3_limit(db, SQLITE_LIMIT_VARIABLE_NUMBER, 100); + sqlite3_limit(db, SQLITE_LIMIT_TRIGGER_DEPTH, 1); + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, 1); + + /* Default is now 4k pages, so allow 500MB */ + err = sqlite3_exec(db, tal_fmt(tmpctx, "PRAGMA max_page_count = %zu;", + max_dbmem / 4096), + NULL, NULL, &errmsg); + if (err != SQLITE_OK) + plugin_err(plugin, "Could not set max_page_count: %s", errmsg); + + return db; +} + +static bool has_table_desc(struct table_desc **tables, struct table_desc *t) +{ + for (size_t i = 0; i < tal_count(tables); i++) { + if (tables[i] == t) + return true; + } + return false; +} + +static int sqlite_authorize(void *dbq_, int code, + const char *a, + const char *b, + const char *c, + const char *d) +{ + struct db_query *dbq = dbq_; + + /* You can do select statements */ + if (code == SQLITE_SELECT) + return SQLITE_OK; + + /* You can do a column read: takes a table name */ + if (code == SQLITE_READ) { + struct table_desc *td = strmap_get(&tablemap, a); + if (!td) { + dbq->authfail = tal_fmt(dbq, "Unknown table %s", a); + return SQLITE_DENY; + } + if (!has_table_desc(dbq->tables, td)) + tal_arr_expand(&dbq->tables, td); + return SQLITE_OK; + } + + /* See https://www.sqlite.org/c3ref/c_alter_table.html to decode these! */ + dbq->authfail = tal_fmt(dbq, "Unauthorized: %u arg1=%s arg2=%s dbname=%s caller=%s", + code, + a ? a : "(none)", + b ? b : "(none)", + c ? c : "(none)", + d ? d : "(none)"); + return SQLITE_DENY; +} + +static struct command_result *refresh_complete(struct command *cmd, + struct db_query *dbq) +{ + char *errmsg; + int err, num_cols; + size_t num_rows; + struct json_stream *ret; + + num_cols = sqlite3_column_count(dbq->stmt); + + /* We normally hit an error immediately, so return a simple error then */ + ret = NULL; + num_rows = 0; + errmsg = NULL; + + while ((err = sqlite3_step(dbq->stmt)) == SQLITE_ROW) { + if (!ret) { + ret = jsonrpc_stream_success(cmd); + json_array_start(ret, "rows"); + } + json_array_start(ret, NULL); + for (size_t i = 0; i < num_cols; i++) { + /* The returned value is one of + * SQLITE_INTEGER, SQLITE_FLOAT, SQLITE_TEXT, + * SQLITE_BLOB, or SQLITE_NULL */ + switch (sqlite3_column_type(dbq->stmt, i)) { + case SQLITE_INTEGER: { + s64 v = sqlite3_column_int64(dbq->stmt, i); + json_add_s64(ret, NULL, v); + break; + } + case SQLITE_FLOAT: { + double v = sqlite3_column_double(dbq->stmt, i); + json_add_primitive_fmt(ret, NULL, "%f", v); + break; + } + case SQLITE_TEXT: { + const char *c = (char *)sqlite3_column_text(dbq->stmt, i); + if (!utf8_check(c, strlen(c))) { + json_add_str_fmt(ret, NULL, + "INVALID UTF-8 STRING %s", + tal_hexstr(tmpctx, c, strlen(c))); + errmsg = tal_fmt(cmd, "Invalid UTF-8 string row %zu column %zu", + num_rows, i); + } else + json_add_string(ret, NULL, c); + break; + } + case SQLITE_BLOB: + json_add_hex(ret, NULL, + sqlite3_column_blob(dbq->stmt, i), + sqlite3_column_bytes(dbq->stmt, i)); + break; + case SQLITE_NULL: + json_add_primitive(ret, NULL, "null"); + break; + default: + errmsg = tal_fmt(cmd, "Unknown column type %i in row %zu column %zu", + sqlite3_column_type(dbq->stmt, i), + num_rows, i); + } + } + json_array_end(ret); + num_rows++; + } + if (err != SQLITE_DONE) + errmsg = tal_fmt(cmd, "Executing statement: %s", + sqlite3_errmsg(db)); + + sqlite3_finalize(dbq->stmt); + + + /* OK, did we hit some error during? Simple if we didn't + * already start answering! */ + if (errmsg) { + if (!ret) + return command_fail(cmd, LIGHTNINGD, "%s", errmsg); + + /* Otherwise, add it as a warning */ + json_array_end(ret); + json_add_string(ret, "warning_db_failure", errmsg); + } else { + /* Empty result is possible, OK. */ + if (!ret) { + ret = jsonrpc_stream_success(cmd); + json_array_start(ret, "rows"); + } + json_array_end(ret); + } + return command_finished(cmd, ret); +} + +/* Recursion */ +static struct command_result *refresh_tables(struct command *cmd, + struct db_query *dbq); + +static struct command_result *list_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct db_query *dbq) +{ + const struct table_desc *td = dbq->tables[0]; + size_t i; + const jsmntok_t *t, *arr = json_get_member(buf, result, td->name); + int err; + sqlite3_stmt *stmt; + char *errmsg; + + /* FIXME: this is where a wait / pagination API is useful! */ + err = sqlite3_exec(db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name), + NULL, NULL, &errmsg); + if (err != SQLITE_OK) { + return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s", + td->name, errmsg); + } + + err = sqlite3_prepare_v2(db, td->update_stmt, -1, &stmt, NULL); + if (err != SQLITE_OK) { + return command_fail(cmd, LIGHTNINGD, "preparing '%s' failed: %s", + td->update_stmt, + sqlite3_errmsg(db)); + } + + json_for_each_arr(i, t, arr) { + size_t c; + /* FIXME: This is O(n^2): hash td->columns and look up + * the other way. */ + for (c = 0; c < tal_count(td->columns); c++) { + const jsmntok_t *col = json_get_member(buf, t, td->columns[c]); + if (!col) + sqlite3_bind_null(stmt, c + 1); + else { + u64 val64; + struct amount_msat valmsat; + u8 *valhex; + double valdouble; + bool valbool; + + switch (td->fieldtypes[c]) { + case FIELD_U8: + case FIELD_U16: + case FIELD_U32: + case FIELD_U64: + case FIELD_INTEGER: + if (!json_to_u64(buf, col, &val64)) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not a u64: %.*s", + c, i, + json_tok_full_len(col), + json_tok_full(buf, col)); + } + sqlite3_bind_int64(stmt, c + 1, val64); + break; + case FIELD_BOOL: + if (!json_to_bool(buf, col, &valbool)) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not a boolean: %.*s", + c, i, + json_tok_full_len(col), + json_tok_full(buf, col)); + } + sqlite3_bind_int(stmt, c + 1, valbool); + break; + case FIELD_NUMBER: + if (!json_to_double(buf, col, &valdouble)) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not a double: %.*s", + c, i, + json_tok_full_len(col), + json_tok_full(buf, col)); + } + sqlite3_bind_double(stmt, c + 1, valdouble); + break; + case FIELD_MSAT: + if (!json_to_msat(buf, col, &valmsat)) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not an msat: %.*s", + c, i, + json_tok_full_len(col), + json_tok_full(buf, col)); + } + sqlite3_bind_int64(stmt, c + 1, valmsat.millisatoshis /* Raw: db */); + break; + case FIELD_SCID: + case FIELD_STRING: + sqlite3_bind_text(stmt, c + 1, buf + col->start, + col->end - col->start, SQLITE_STATIC); + break; + case FIELD_HEX: + case FIELD_HASH: + case FIELD_SECRET: + case FIELD_PUBKEY: + case FIELD_TXID: + valhex = json_tok_bin_from_hex(tmpctx, buf, col); + if (!valhex) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not valid hex: %.*s", + c, i, + json_tok_full_len(col), + json_tok_full(buf, col)); + } + sqlite3_bind_blob(stmt, c + 1, valhex, tal_count(valhex), + SQLITE_STATIC); + break; + } + } + } + err = sqlite3_step(stmt); + + if (err != SQLITE_DONE) { + const char *emsg = sqlite3_errmsg(db); + sqlite3_finalize(stmt); + return command_fail(cmd, LIGHTNINGD, + "Error executing %s on column %zu row %zu: %s", + td->update_stmt, + c, i, emsg); + } + sqlite3_reset(stmt); + } + sqlite3_finalize(stmt); + + /* Remove that, iterate */ + tal_arr_remove(&dbq->tables, 0); + return refresh_tables(cmd, dbq); +} + +static struct command_result *refresh_tables(struct command *cmd, + struct db_query *dbq) +{ + struct out_req *req; + const struct table_desc *td; + + if (tal_count(dbq->tables) == 0) + return refresh_complete(cmd, dbq); + + td = dbq->tables[0]; + req = jsonrpc_request_start(cmd->plugin, cmd, + tal_fmt(tmpctx, "list%s", td->name), + list_done, forward_error, + dbq); + return send_outreq(cmd->plugin, req); +} + +static struct command_result *json_sql(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct db_query *dbq = tal(cmd, struct db_query); + const char *query; + int err; + + if (!param(cmd, buffer, params, + p_req("query", param_string, &query), + NULL)) + return command_param_failed(); + + dbq->tables = tal_arr(dbq, struct table_desc *, 0); + dbq->authfail = NULL; + + /* This both checks we're not altering, *and* tells us what + * tables to refresh. */ + err = sqlite3_set_authorizer(db, sqlite_authorize, dbq); + if (err != SQLITE_OK) { + plugin_err(cmd->plugin, "Could not set authorizer: %s", + sqlite3_errmsg(db)); + } + + err = sqlite3_prepare_v2(db, query, -1, &dbq->stmt, NULL); + sqlite3_set_authorizer(db, NULL, NULL); + + if (err != SQLITE_OK) { + char *errmsg = tal_fmt(tmpctx, "query failed with %s", sqlite3_errmsg(db)); + if (dbq->authfail) + tal_append_fmt(&errmsg, " (%s)", dbq->authfail); + return command_fail(cmd, LIGHTNINGD, "%s", errmsg); + } + + return refresh_tables(cmd, dbq); +} + +static void init_tablemap(struct plugin *plugin) +{ + struct table_desc *td; + char *create_stmt; + int err; + char *errmsg; + + strmap_init(&tablemap); + + /* FIXME: Load from schemas! */ + td = tal(NULL, struct table_desc); + td->name = "forwards"; + td->columns = tal_arr(td, const char *, 11); + td->fieldtypes = tal_arr(td, enum fieldtype, 11); + td->columns[0] = "in_htlc_id"; + td->fieldtypes[0] = FIELD_U64; + td->columns[1] = "in_channel"; + td->fieldtypes[1] = FIELD_SCID; + td->columns[2] = "in_msat"; + td->fieldtypes[2] = FIELD_MSAT; + td->columns[3] = "status"; + td->fieldtypes[3] = FIELD_STRING; + td->columns[4] = "received_time"; + td->fieldtypes[4] = FIELD_NUMBER; + td->columns[5] = "out_channel"; + td->fieldtypes[5] = FIELD_SCID; + td->columns[6] = "out_htlc_id"; + td->fieldtypes[6] = FIELD_U64; + td->columns[7] = "style"; + td->fieldtypes[7] = FIELD_STRING; + td->columns[8] = "fee_msat"; + td->fieldtypes[8] = FIELD_MSAT; + td->columns[9] = "out_msat"; + td->fieldtypes[9] = FIELD_MSAT; + td->columns[10] = "resolved_time"; + td->fieldtypes[10] = FIELD_NUMBER; + + /* FIXME: Primary key from schema? */ + create_stmt = tal_fmt(tmpctx, "CREATE TABLE %s (", td->name); + td->update_stmt = tal_fmt(td, "INSERT INTO %s VALUES (", td->name); + for (size_t i = 0; i < tal_count(td->columns); i++) { + tal_append_fmt(&td->update_stmt, "%s?", + i == 0 ? "" : ","); + tal_append_fmt(&create_stmt, "%s%s %s", + i == 0 ? "" : ",", + td->columns[i], + fieldtypemap[td->fieldtypes[i]].sqltype); + } + tal_append_fmt(&create_stmt, ");"); + tal_append_fmt(&td->update_stmt, ");"); + + err = sqlite3_exec(db, create_stmt, NULL, NULL, &errmsg); + if (err != SQLITE_OK) + plugin_err(plugin, "Could not create %s: %s", td->name, errmsg); + + strmap_add(&tablemap, td->name, td); +} + +#if DEVELOPER +static void memleak_mark_tablemap(struct plugin *p, struct htable *memtable) +{ + memleak_ptr(memtable, dbfilename); + memleak_scan_strmap(memtable, &tablemap); +} +#endif + +static const char *init(struct plugin *plugin, + const char *buf UNUSED, const jsmntok_t *config UNUSED) +{ + db = sqlite_setup(plugin); + init_tablemap(plugin); + +#if DEVELOPER + plugin_set_memleak_handler(plugin, memleak_mark_tablemap); +#endif + return NULL; +} + +static const struct plugin_command commands[] = { { + "sql", + "misc", + "Run {query} and return result", + "This is the greatest plugin command ever!", + json_sql, + }, +}; + +int main(int argc, char *argv[]) +{ + setup_locale(); + plugin_main(argv, init, PLUGIN_RESTARTABLE, true, NULL, commands, ARRAY_SIZE(commands), + NULL, 0, NULL, 0, NULL, 0, + plugin_option("sqlfilename", + "string", + "Use on-disk sqlite3 file instead of in memory (e.g. debugging)", + charp_option, &dbfilename), + NULL); +} diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 04d1cfbe0ffd..2e20dcd8d350 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3275,3 +3275,22 @@ def test_block_added_notifications(node_factory, bitcoind): sync_blockheight(bitcoind, [l2]) ret = l2.rpc.call("blockscatched") assert len(ret) == 3 and ret[1] == next_l2_base + 1 and ret[2] == next_l2_base + 2 + + +def test_sql(node_factory, bitcoind): + l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True) + + ret = l2.rpc.sql("SELECT * FROM forwards;") + assert ret == {'rows': []} + + # This should create a forward through l2 + l1.rpc.pay(l3.rpc.invoice(amount_msat=12300, label='inv1', description='description')['bolt11']) + + ret = l2.rpc.sql("SELECT in_htlc_id,out_msat,status,out_htlc_id FROM forwards;") + assert only_one(ret['rows'])[0] == 0 + assert only_one(ret['rows'])[1] == 12300 + assert only_one(ret['rows'])[2] == 'settled' + assert only_one(ret['rows'])[3] == 0 + + with pytest.raises(RpcError, match='Unauthorized'): + l2.rpc.sql("DELETE FROM forwards;") From f98d1a8a27fb821828a2ca87b1901cb2869249e2 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:17 +1030 Subject: [PATCH 21/36] plugins/sql: create `struct column` to encode column details. Rather than two arrays "columns" (for names) and "fieldtypes" (for types), use a struct. This makes additions easier for successive patches. Also pull process_json_obj() out of the loop in list_done(). Signed-off-by: Rusty Russell --- plugins/sql.c | 358 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 213 insertions(+), 145 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index 155bac36e1f0..e6625cbd0eaa 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -65,6 +65,11 @@ static const struct fieldtypemap fieldtypemap[] = { { "short_channel_id", "TEXT" }, /* FIELD_SCID */ }; +struct column { + const char *name; + enum fieldtype ftype; +}; + struct db_query { sqlite3_stmt *stmt; struct table_desc **tables; @@ -74,9 +79,8 @@ struct db_query { struct table_desc { /* e.g. peers for listpeers */ const char *name; - const char **columns; + struct column *columns; char *update_stmt; - enum fieldtype *fieldtypes; }; static STRMAP(struct table_desc *) tablemap; static size_t max_dbmem = 500000000; @@ -272,25 +276,137 @@ static struct command_result *refresh_complete(struct command *cmd, static struct command_result *refresh_tables(struct command *cmd, struct db_query *dbq); -static struct command_result *list_done(struct command *cmd, - const char *buf, - const jsmntok_t *result, - struct db_query *dbq) +static struct command_result *one_refresh_done(struct command *cmd, + struct db_query *dbq) +{ + /* Remove that, iterate */ + tal_arr_remove(&dbq->tables, 0); + return refresh_tables(cmd, dbq); +} + +/* Returns NULL on success, otherwise has failed cmd. */ +static struct command_result *process_json_obj(struct command *cmd, + const char *buf, + const jsmntok_t *t, + const struct table_desc *td, + size_t row, + const u64 *rowid, + size_t *sqloff, + sqlite3_stmt *stmt) +{ + int err; + + /* FIXME: This is O(n^2): hash td->columns and look up the other way. */ + for (size_t i = 0; i < tal_count(td->columns); i++) { + const struct column *col = &td->columns[i]; + const jsmntok_t *coltok; + + if (!t) + coltok = NULL; + else + coltok = json_get_member(buf, t, col->name); + + if (!coltok) + sqlite3_bind_null(stmt, (*sqloff)++); + else { + u64 val64; + struct amount_msat valmsat; + u8 *valhex; + double valdouble; + bool valbool; + + switch (col->ftype) { + case FIELD_U8: + case FIELD_U16: + case FIELD_U32: + case FIELD_U64: + case FIELD_INTEGER: + if (!json_to_u64(buf, coltok, &val64)) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not a u64: %.*s", + i, row, + json_tok_full_len(coltok), + json_tok_full(buf, coltok)); + } + sqlite3_bind_int64(stmt, (*sqloff)++, val64); + break; + case FIELD_BOOL: + if (!json_to_bool(buf, coltok, &valbool)) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not a boolean: %.*s", + i, row, + json_tok_full_len(coltok), + json_tok_full(buf, coltok)); + } + sqlite3_bind_int(stmt, (*sqloff)++, valbool); + break; + case FIELD_NUMBER: + if (!json_to_double(buf, coltok, &valdouble)) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not a double: %.*s", + i, row, + json_tok_full_len(coltok), + json_tok_full(buf, coltok)); + } + sqlite3_bind_double(stmt, (*sqloff)++, valdouble); + break; + case FIELD_MSAT: + if (!json_to_msat(buf, coltok, &valmsat)) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not an msat: %.*s", + i, row, + json_tok_full_len(coltok), + json_tok_full(buf, coltok)); + } + sqlite3_bind_int64(stmt, (*sqloff)++, valmsat.millisatoshis /* Raw: db */); + break; + case FIELD_SCID: + case FIELD_STRING: + sqlite3_bind_text(stmt, (*sqloff)++, buf + coltok->start, + coltok->end - coltok->start, + SQLITE_STATIC); + break; + case FIELD_HEX: + case FIELD_HASH: + case FIELD_SECRET: + case FIELD_PUBKEY: + case FIELD_TXID: + valhex = json_tok_bin_from_hex(tmpctx, buf, coltok); + if (!valhex) { + return command_fail(cmd, LIGHTNINGD, + "column %zu row %zu not valid hex: %.*s", + i, row, + json_tok_full_len(coltok), + json_tok_full(buf, coltok)); + } + sqlite3_bind_blob(stmt, (*sqloff)++, valhex, tal_count(valhex), + SQLITE_STATIC); + break; + } + } + } + + err = sqlite3_step(stmt); + if (err != SQLITE_DONE) { + return command_fail(cmd, LIGHTNINGD, + "Error executing %s on row %zu: %s", + td->update_stmt, + row, + sqlite3_errmsg(db)); + } + return NULL; +} + +static struct command_result *process_json_list(struct command *cmd, + const char *buf, + const jsmntok_t *result, + const struct table_desc *td) { - const struct table_desc *td = dbq->tables[0]; size_t i; const jsmntok_t *t, *arr = json_get_member(buf, result, td->name); int err; sqlite3_stmt *stmt; - char *errmsg; - - /* FIXME: this is where a wait / pagination API is useful! */ - err = sqlite3_exec(db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name), - NULL, NULL, &errmsg); - if (err != SQLITE_OK) { - return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s", - td->name, errmsg); - } + struct command_result *ret = NULL; err = sqlite3_prepare_v2(db, td->update_stmt, -1, &stmt, NULL); if (err != SQLITE_OK) { @@ -299,124 +415,65 @@ static struct command_result *list_done(struct command *cmd, sqlite3_errmsg(db)); } - json_for_each_arr(i, t, arr) { - size_t c; - /* FIXME: This is O(n^2): hash td->columns and look up - * the other way. */ - for (c = 0; c < tal_count(td->columns); c++) { - const jsmntok_t *col = json_get_member(buf, t, td->columns[c]); - if (!col) - sqlite3_bind_null(stmt, c + 1); - else { - u64 val64; - struct amount_msat valmsat; - u8 *valhex; - double valdouble; - bool valbool; - - switch (td->fieldtypes[c]) { - case FIELD_U8: - case FIELD_U16: - case FIELD_U32: - case FIELD_U64: - case FIELD_INTEGER: - if (!json_to_u64(buf, col, &val64)) { - return command_fail(cmd, LIGHTNINGD, - "column %zu row %zu not a u64: %.*s", - c, i, - json_tok_full_len(col), - json_tok_full(buf, col)); - } - sqlite3_bind_int64(stmt, c + 1, val64); - break; - case FIELD_BOOL: - if (!json_to_bool(buf, col, &valbool)) { - return command_fail(cmd, LIGHTNINGD, - "column %zu row %zu not a boolean: %.*s", - c, i, - json_tok_full_len(col), - json_tok_full(buf, col)); - } - sqlite3_bind_int(stmt, c + 1, valbool); - break; - case FIELD_NUMBER: - if (!json_to_double(buf, col, &valdouble)) { - return command_fail(cmd, LIGHTNINGD, - "column %zu row %zu not a double: %.*s", - c, i, - json_tok_full_len(col), - json_tok_full(buf, col)); - } - sqlite3_bind_double(stmt, c + 1, valdouble); - break; - case FIELD_MSAT: - if (!json_to_msat(buf, col, &valmsat)) { - return command_fail(cmd, LIGHTNINGD, - "column %zu row %zu not an msat: %.*s", - c, i, - json_tok_full_len(col), - json_tok_full(buf, col)); - } - sqlite3_bind_int64(stmt, c + 1, valmsat.millisatoshis /* Raw: db */); - break; - case FIELD_SCID: - case FIELD_STRING: - sqlite3_bind_text(stmt, c + 1, buf + col->start, - col->end - col->start, SQLITE_STATIC); - break; - case FIELD_HEX: - case FIELD_HASH: - case FIELD_SECRET: - case FIELD_PUBKEY: - case FIELD_TXID: - valhex = json_tok_bin_from_hex(tmpctx, buf, col); - if (!valhex) { - return command_fail(cmd, LIGHTNINGD, - "column %zu row %zu not valid hex: %.*s", - c, i, - json_tok_full_len(col), - json_tok_full(buf, col)); - } - sqlite3_bind_blob(stmt, c + 1, valhex, tal_count(valhex), - SQLITE_STATIC); - break; - } - } - } - err = sqlite3_step(stmt); - - if (err != SQLITE_DONE) { - const char *emsg = sqlite3_errmsg(db); - sqlite3_finalize(stmt); - return command_fail(cmd, LIGHTNINGD, - "Error executing %s on column %zu row %zu: %s", - td->update_stmt, - c, i, emsg); - } + json_for_each_arr(i, t, arr) { + /* sqlite3 columns are 1-based! */ + size_t off = 1; + ret = process_json_obj(cmd, buf, t, td, i, NULL, &off, stmt); + if (ret) + break; sqlite3_reset(stmt); } sqlite3_finalize(stmt); + return ret; +} - /* Remove that, iterate */ - tal_arr_remove(&dbq->tables, 0); - return refresh_tables(cmd, dbq); +static struct command_result *default_list_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct db_query *dbq) +{ + const struct table_desc *td = dbq->tables[0]; + struct command_result *ret; + int err; + char *errmsg; + + /* FIXME: this is where a wait / pagination API is useful! */ + err = sqlite3_exec(db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name), + NULL, NULL, &errmsg); + if (err != SQLITE_OK) { + return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s", + td->name, errmsg); + } + + ret = process_json_list(cmd, buf, result, td); + if (ret) + return ret; + + return one_refresh_done(cmd, dbq); +} + +static struct command_result *default_refresh(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq) +{ + struct out_req *req; + req = jsonrpc_request_start(cmd->plugin, cmd, + tal_fmt(tmpctx, "list%s", td->name), + default_list_done, forward_error, + dbq); + return send_outreq(cmd->plugin, req); } static struct command_result *refresh_tables(struct command *cmd, struct db_query *dbq) { - struct out_req *req; const struct table_desc *td; if (tal_count(dbq->tables) == 0) return refresh_complete(cmd, dbq); td = dbq->tables[0]; - req = jsonrpc_request_start(cmd->plugin, cmd, - tal_fmt(tmpctx, "list%s", td->name), - list_done, forward_error, - dbq); - return send_outreq(cmd->plugin, req); + return default_refresh(cmd, td, dbq); } static struct command_result *json_sql(struct command *cmd, @@ -462,36 +519,47 @@ static void init_tablemap(struct plugin *plugin) char *create_stmt; int err; char *errmsg; + struct column col; strmap_init(&tablemap); /* FIXME: Load from schemas! */ td = tal(NULL, struct table_desc); td->name = "forwards"; - td->columns = tal_arr(td, const char *, 11); - td->fieldtypes = tal_arr(td, enum fieldtype, 11); - td->columns[0] = "in_htlc_id"; - td->fieldtypes[0] = FIELD_U64; - td->columns[1] = "in_channel"; - td->fieldtypes[1] = FIELD_SCID; - td->columns[2] = "in_msat"; - td->fieldtypes[2] = FIELD_MSAT; - td->columns[3] = "status"; - td->fieldtypes[3] = FIELD_STRING; - td->columns[4] = "received_time"; - td->fieldtypes[4] = FIELD_NUMBER; - td->columns[5] = "out_channel"; - td->fieldtypes[5] = FIELD_SCID; - td->columns[6] = "out_htlc_id"; - td->fieldtypes[6] = FIELD_U64; - td->columns[7] = "style"; - td->fieldtypes[7] = FIELD_STRING; - td->columns[8] = "fee_msat"; - td->fieldtypes[8] = FIELD_MSAT; - td->columns[9] = "out_msat"; - td->fieldtypes[9] = FIELD_MSAT; - td->columns[10] = "resolved_time"; - td->fieldtypes[10] = FIELD_NUMBER; + td->columns = tal_arr(td, struct column, 0); + col.name = "in_htlc_id"; + col.ftype = FIELD_U64; + tal_arr_expand(&td->columns, col); + col.name = "in_channel"; + col.ftype = FIELD_SCID; + tal_arr_expand(&td->columns, col); + col.name = "in_msat"; + col.ftype = FIELD_MSAT; + tal_arr_expand(&td->columns, col); + col.name = "status"; + col.ftype = FIELD_STRING; + tal_arr_expand(&td->columns, col); + col.name = "received_time"; + col.ftype = FIELD_NUMBER; + tal_arr_expand(&td->columns, col); + col.name = "out_channel"; + col.ftype = FIELD_SCID; + tal_arr_expand(&td->columns, col); + col.name = "out_htlc_id"; + col.ftype = FIELD_U64; + tal_arr_expand(&td->columns, col); + col.name = "style"; + col.ftype = FIELD_STRING; + tal_arr_expand(&td->columns, col); + col.name = "fee_msat"; + col.ftype = FIELD_MSAT; + tal_arr_expand(&td->columns, col); + col.name = "out_msat"; + col.ftype = FIELD_MSAT; + tal_arr_expand(&td->columns, col); + col.name = "resolved_time"; + col.ftype = FIELD_NUMBER; + tal_arr_expand(&td->columns, col); /* FIXME: Primary key from schema? */ create_stmt = tal_fmt(tmpctx, "CREATE TABLE %s (", td->name); @@ -501,8 +569,8 @@ static void init_tablemap(struct plugin *plugin) i == 0 ? "" : ","); tal_append_fmt(&create_stmt, "%s%s %s", i == 0 ? "" : ",", - td->columns[i], - fieldtypemap[td->fieldtypes[i]].sqltype); + td->columns[i].name, + fieldtypemap[td->columns[i].ftype].sqltype); } tal_append_fmt(&create_stmt, ");"); tal_append_fmt(&td->update_stmt, ");"); From bea8babac843780cbc067a1ce1f73e12bc3f6ab3 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:17 +1030 Subject: [PATCH 22/36] plugins/sql: rework to parse schemas. This requires us to rename "index" fields, rename fields if we have a sub-object, and create sub-tables if we have an array, and handle the fact that some listX commands don't contain array X (listsendpays contains "payments"). Signed-off-by: Rusty Russell --- plugins/Makefile | 14 ++ plugins/sql.c | 414 ++++++++++++++++++++++++++++++++++++------- tests/test_plugin.py | 33 +++- 3 files changed, 395 insertions(+), 66 deletions(-) diff --git a/plugins/Makefile b/plugins/Makefile index ffbdd0deddc3..b1a4e8363ec3 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -204,6 +204,20 @@ plugins/fetchinvoice: $(PLUGIN_FETCHINVOICE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_CO plugins/funder: bitcoin/psbt.o common/psbt_open.o $(PLUGIN_FUNDER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) +# This covers all the low-level list RPCs which return simple arrays +SQL_LISTRPCS := listchannels listforwards listhtlcs listinvoices listnodes listoffers listpeers listtransactions listsendpays +SQL_LISTRPCS_SCHEMAS := $(foreach l,$(SQL_LISTRPCS),doc/schemas/$l.schema.json) +# We squeeze: +# descriptions (we don't need) +# fields with no members (we don't need) +# whitespace +# We can't simply *remove* fields, since the extra comma left over can +# make invalid JSON. Grr! +# But these simple removals drop us from 100k to 29k. +plugins/sql-schema_gen.h: plugins/Makefile $(SQL_LISTRPCS_SCHEMAS) + @$(call VERBOSE,GEN $@, (SEP=""; echo -n '"{'; for f in $(SQL_LISTRPCS); do echo -n "$$SEP\\\"$$f\\\":"; sed -e s/\"description\":\ *\".\*\"/\"\":\"\"/ -e s/\".*\":\ *{}/\"\":{}/ -e s/\"/\\\\\"/g < doc/schemas/$$f.schema.json | tr -d ' \n'; SEP=","; done; echo '}"') > $@) + +plugins/sql.o: plugins/sql-schema_gen.h plugins/sql: $(PLUGIN_SQL_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) # Generated from PLUGINS definition in plugins/Makefile diff --git a/plugins/sql.c b/plugins/sql.c index e6625cbd0eaa..f07f68cd86fe 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1,6 +1,7 @@ /* Brilliant or insane? You decide! */ #include "config.h" #include +#include #include #include #include @@ -9,8 +10,12 @@ #include #include +/* Minimized schemas. C23 #embed, Where Art Thou? */ +static const char schemas[] = + #include "sql-schema_gen.h" + ; + /* TODO: - * 1. Generate from schemas. * 2. Refresh time in API. * 3. Colnames API to return dict. * 4. sql-schemas command. @@ -66,8 +71,12 @@ static const struct fieldtypemap fieldtypemap[] = { }; struct column { - const char *name; + /* We rename some fields to avoid sql keywords! */ + const char *dbname, *jsonname; enum fieldtype ftype; + + /* If this is actually a subtable: */ + struct table_desc *sub; }; struct db_query { @@ -77,16 +86,38 @@ struct db_query { }; struct table_desc { + /* e.g. listpeers */ + const char *cmdname; /* e.g. peers for listpeers */ const char *name; + /* e.g. "payments" for listsendpays */ + const char *arrname; struct column *columns; char *update_stmt; + /* If we are a subtable */ + struct table_desc *parent; + /* Is this a sub object (otherwise, subarray if parent is true) */ + bool is_subobject; + /* function to refresh it. */ + struct command_result *(*refresh)(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq); }; static STRMAP(struct table_desc *) tablemap; static size_t max_dbmem = 500000000; static struct sqlite3 *db; static const char *dbfilename; +static enum fieldtype find_fieldtype(const jsmntok_t *name) +{ + for (size_t i = 0; i < ARRAY_SIZE(fieldtypemap); i++) { + if (json_tok_streq(schemas, name, fieldtypemap[i].name)) + return i; + } + errx(1, "Unknown JSON type %.*s", + name->end - name->start, schemas + name->start); +} + static struct sqlite3 *sqlite_setup(struct plugin *plugin) { int err; @@ -165,6 +196,9 @@ static int sqlite_authorize(void *dbq_, int code, dbq->authfail = tal_fmt(dbq, "Unknown table %s", a); return SQLITE_DENY; } + /* If it has a parent, we refresh that instead */ + while (td->parent) + td = td->parent; if (!has_table_desc(dbq->tables, td)) tal_arr_expand(&dbq->tables, td); return SQLITE_OK; @@ -284,6 +318,13 @@ static struct command_result *one_refresh_done(struct command *cmd, return refresh_tables(cmd, dbq); } +/* Mutual recursion */ +static struct command_result *process_json_list(struct command *cmd, + const char *buf, + const jsmntok_t *arr, + const u64 *rowid, + const struct table_desc *td); + /* Returns NULL on success, otherwise has failed cmd. */ static struct command_result *process_json_obj(struct command *cmd, const char *buf, @@ -295,20 +336,47 @@ static struct command_result *process_json_obj(struct command *cmd, sqlite3_stmt *stmt) { int err; + u64 parent_rowid; + + /* Subtables have row, arrindex as first two columns. */ + if (rowid) { + sqlite3_bind_int64(stmt, (*sqloff)++, *rowid); + sqlite3_bind_int64(stmt, (*sqloff)++, row); + } /* FIXME: This is O(n^2): hash td->columns and look up the other way. */ for (size_t i = 0; i < tal_count(td->columns); i++) { const struct column *col = &td->columns[i]; const jsmntok_t *coltok; + if (col->sub) { + struct command_result *ret; + /* Handle sub-tables below: we need rowid! */ + if (!col->sub->is_subobject) + continue; + + coltok = json_get_member(buf, t, col->jsonname); + ret = process_json_obj(cmd, buf, coltok, col->sub, row, NULL, + sqloff, stmt); + if (ret) + return ret; + continue; + } + + /* This can happen if subobject does not exist in output! */ if (!t) coltok = NULL; else - coltok = json_get_member(buf, t, col->name); - - if (!coltok) + coltok = json_get_member(buf, t, col->jsonname); + + if (!coltok) { + if (td->parent) + plugin_log(cmd->plugin, LOG_DBG, + "Did not find json %s for %s in %.*s", + col->jsonname, td->name, + t ? json_tok_full_len(t) : 4, t ? json_tok_full(buf, t): "NULL"); sqlite3_bind_null(stmt, (*sqloff)++); - else { + } else { u64 val64; struct amount_msat valmsat; u8 *valhex; @@ -386,6 +454,10 @@ static struct command_result *process_json_obj(struct command *cmd, } } + /* Sub objects get folded into parent's SQL */ + if (td->parent && td->is_subobject) + return NULL; + err = sqlite3_step(stmt); if (err != SQLITE_DONE) { return command_fail(cmd, LIGHTNINGD, @@ -394,16 +466,37 @@ static struct command_result *process_json_obj(struct command *cmd, row, sqlite3_errmsg(db)); } + + /* Now we have rowid, we can insert into any subtables. */ + parent_rowid = sqlite3_last_insert_rowid(db); + for (size_t i = 0; i < tal_count(td->columns); i++) { + const struct column *col = &td->columns[i]; + const jsmntok_t *coltok; + struct command_result *ret; + + if (!col->sub || col->sub->is_subobject) + continue; + + coltok = json_get_member(buf, t, col->jsonname); + if (!coltok) + continue; + + ret = process_json_list(cmd, buf, coltok, &parent_rowid, col->sub); + if (ret) + return ret; + } return NULL; } +/* A list, such as in the top-level reply, or for a sub-table */ static struct command_result *process_json_list(struct command *cmd, const char *buf, - const jsmntok_t *result, + const jsmntok_t *arr, + const u64 *rowid, const struct table_desc *td) { size_t i; - const jsmntok_t *t, *arr = json_get_member(buf, result, td->name); + const jsmntok_t *t; int err; sqlite3_stmt *stmt; struct command_result *ret = NULL; @@ -415,10 +508,10 @@ static struct command_result *process_json_list(struct command *cmd, sqlite3_errmsg(db)); } - json_for_each_arr(i, t, arr) { + json_for_each_arr(i, t, arr) { /* sqlite3 columns are 1-based! */ size_t off = 1; - ret = process_json_obj(cmd, buf, t, td, i, NULL, &off, stmt); + ret = process_json_obj(cmd, buf, t, td, i, rowid, &off, stmt); if (ret) break; sqlite3_reset(stmt); @@ -427,6 +520,17 @@ static struct command_result *process_json_list(struct command *cmd, return ret; } +/* Process top-level JSON result object */ +static struct command_result *process_json_result(struct command *cmd, + const char *buf, + const jsmntok_t *result, + const struct table_desc *td) +{ + return process_json_list(cmd, buf, + json_get_member(buf, result, td->arrname), + NULL, td); +} + static struct command_result *default_list_done(struct command *cmd, const char *buf, const jsmntok_t *result, @@ -445,7 +549,7 @@ static struct command_result *default_list_done(struct command *cmd, td->name, errmsg); } - ret = process_json_list(cmd, buf, result, td); + ret = process_json_result(cmd, buf, result, td); if (ret) return ret; @@ -457,8 +561,7 @@ static struct command_result *default_refresh(struct command *cmd, struct db_query *dbq) { struct out_req *req; - req = jsonrpc_request_start(cmd->plugin, cmd, - tal_fmt(tmpctx, "list%s", td->name), + req = jsonrpc_request_start(cmd->plugin, cmd, td->cmdname, default_list_done, forward_error, dbq); return send_outreq(cmd->plugin, req); @@ -473,7 +576,7 @@ static struct command_result *refresh_tables(struct command *cmd, return refresh_complete(cmd, dbq); td = dbq->tables[0]; - return default_refresh(cmd, td, dbq); + return td->refresh(cmd, dbq->tables[0], dbq); } static struct command_result *json_sql(struct command *cmd, @@ -513,64 +616,74 @@ static struct command_result *json_sql(struct command *cmd, return refresh_tables(cmd, dbq); } -static void init_tablemap(struct plugin *plugin) +static bool ignore_column(const struct table_desc *td, const jsmntok_t *t) +{ + /* We don't use peers.log, since it'll always be empty unless we were to + * ask for it in listpeers, and it's not very useful. */ + if (streq(td->name, "peers") && json_tok_streq(schemas, t, "log")) + return true; + /* FIXME: peers.netaddr is an array of strings, which we don't handle. */ + if (streq(td->name, "peers") && json_tok_streq(schemas, t, "netaddr")) + return true; + /* FIXME: peers.channels.features is an array of strings, which we don't handle. */ + if (streq(td->name, "peers_channels") && json_tok_streq(schemas, t, "features")) + return true; + if (streq(td->name, "peers_channels") && json_tok_streq(schemas, t, "status")) + return true; + return false; +} + +/* Creates sql statements, initializes table, adds to tablemap */ +static void finish_td(struct plugin *plugin, struct table_desc *td) { - struct table_desc *td; char *create_stmt; int err; char *errmsg; - struct column col; + const char *sep = ""; - strmap_init(&tablemap); - - /* FIXME: Load from schemas! */ - td = tal(NULL, struct table_desc); - td->name = "forwards"; - td->columns = tal_arr(td, struct column, 0); - col.name = "in_htlc_id"; - col.ftype = FIELD_U64; - tal_arr_expand(&td->columns, col); - col.name = "in_channel"; - col.ftype = FIELD_SCID; - tal_arr_expand(&td->columns, col); - col.name = "in_msat"; - col.ftype = FIELD_MSAT; - tal_arr_expand(&td->columns, col); - col.name = "status"; - col.ftype = FIELD_STRING; - tal_arr_expand(&td->columns, col); - col.name = "received_time"; - col.ftype = FIELD_NUMBER; - tal_arr_expand(&td->columns, col); - col.name = "out_channel"; - col.ftype = FIELD_SCID; - tal_arr_expand(&td->columns, col); - col.name = "out_htlc_id"; - col.ftype = FIELD_U64; - tal_arr_expand(&td->columns, col); - col.name = "style"; - col.ftype = FIELD_STRING; - tal_arr_expand(&td->columns, col); - col.name = "fee_msat"; - col.ftype = FIELD_MSAT; - tal_arr_expand(&td->columns, col); - col.name = "out_msat"; - col.ftype = FIELD_MSAT; - tal_arr_expand(&td->columns, col); - col.name = "resolved_time"; - col.ftype = FIELD_NUMBER; - tal_arr_expand(&td->columns, col); + /* subobject are separate at JSON level, folded at db level! */ + if (td->is_subobject) + return; /* FIXME: Primary key from schema? */ create_stmt = tal_fmt(tmpctx, "CREATE TABLE %s (", td->name); td->update_stmt = tal_fmt(td, "INSERT INTO %s VALUES (", td->name); + + /* If we're a child array, we reference the parent column */ + if (td->parent) { + tal_append_fmt(&create_stmt, + "row INTEGER REFERENCES %s(rowid) ON DELETE CASCADE," + " arrindex INTEGER", + td->parent->name); + tal_append_fmt(&td->update_stmt, "?,?"); + sep = ","; + } + for (size_t i = 0; i < tal_count(td->columns); i++) { - tal_append_fmt(&td->update_stmt, "%s?", - i == 0 ? "" : ","); + const struct column *col = &td->columns[i]; + + if (col->sub) { + /* sub-arrays are a completely separate table. */ + if (!col->sub->is_subobject) + continue; + /* sub-objects are folded into this table. */ + for (size_t j = 0; j < tal_count(col->sub->columns); j++) { + const struct column *subcol = &col->sub->columns[j]; + tal_append_fmt(&td->update_stmt, "%s?", sep); + tal_append_fmt(&create_stmt, "%s%s %s", + sep, + subcol->dbname, + fieldtypemap[subcol->ftype].sqltype); + sep = ","; + } + continue; + } + tal_append_fmt(&td->update_stmt, "%s?", sep); tal_append_fmt(&create_stmt, "%s%s %s", - i == 0 ? "" : ",", - td->columns[i].name, - fieldtypemap[td->columns[i].ftype].sqltype); + sep, + col->dbname, + fieldtypemap[col->ftype].sqltype); + sep = ","; } tal_append_fmt(&create_stmt, ");"); tal_append_fmt(&td->update_stmt, ");"); @@ -580,6 +693,185 @@ static void init_tablemap(struct plugin *plugin) plugin_err(plugin, "Could not create %s: %s", td->name, errmsg); strmap_add(&tablemap, td->name, td); + + /* Now do any children */ + for (size_t i = 0; i < tal_count(td->columns); i++) { + const struct column *col = &td->columns[i]; + if (col->sub) + finish_td(plugin, col->sub); + } +} + +/* Don't use SQL keywords as column names: sure, you can use quotes, + * but it's a PITA. */ +static const char *db_column_name(const tal_t *ctx, + const struct table_desc *td, + const jsmntok_t *nametok) +{ + const char *name = json_strdup(tmpctx, schemas, nametok); + + if (streq(name, "index")) + name = tal_strdup(tmpctx, "idx"); + + /* Prepend td->name to make column unique in table. */ + if (td->is_subobject) + return tal_fmt(ctx, "%s_%s", td->cmdname, name); + + return tal_steal(ctx, name); +} + +/* Remove 'list', turn - into _ in name */ +static const char *db_table_name(const tal_t *ctx, const char *cmdname) +{ + const char *list = strstr(cmdname, "list"); + char *ret = tal_arr(ctx, char, strlen(cmdname) + 1), *dst = ret; + const char *src = cmdname; + + while (*src) { + if (src == list) + src += strlen("list"); + else if (cisalnum(*src)) + *(dst++) = *(src++); + else { + (*dst++) = '_'; + src++; + } + } + *dst = '\0'; + return ret; +} + +static struct table_desc *new_table_desc(struct table_desc *parent, + const jsmntok_t *cmd, + const jsmntok_t *arrname, + bool is_subobject) +{ + struct table_desc *td; + const char *name; + + td = tal(parent, struct table_desc); + td->cmdname = json_strdup(td, schemas, cmd); + name = db_table_name(tmpctx, td->cmdname); + if (!parent) + td->name = tal_steal(td, name); + else + td->name = tal_fmt(td, "%s_%s", parent->name, name); + td->parent = parent; + td->is_subobject = is_subobject; + td->arrname = json_strdup(td, schemas, arrname); + td->columns = tal_arr(td, struct column, 0); + td->refresh = default_refresh; + return td; +} + +static bool find_column(const struct table_desc *td, + const char *dbname) +{ + for (size_t i = 0; i < tal_count(td->columns); i++) { + if (streq(td->columns[i].dbname, dbname)) + return true; + } + return false; +} + +/* Recursion */ +static void add_table_object(struct table_desc *td, const jsmntok_t *tok); + +static void add_table_properties(struct table_desc *td, + const jsmntok_t *properties) +{ + const jsmntok_t *t; + size_t i; + + json_for_each_obj(i, t, properties) { + const jsmntok_t *type; + struct column col; + + if (ignore_column(td, t)) + continue; + type = json_get_member(schemas, t+1, "type"); + /* Stub properties don't have types, it should exist in + * another branch with actual types, so ignore this */ + if (!type) + continue; + if (json_tok_streq(schemas, type, "array")) { + const jsmntok_t *items; + + items = json_get_member(schemas, t+1, "items"); + type = json_get_member(schemas, items, "type"); + assert(json_tok_streq(schemas, type, "object")); + + col.sub = new_table_desc(td, t, t, false); + add_table_object(col.sub, items); + } else if (json_tok_streq(schemas, type, "object")) { + col.sub = new_table_desc(td, t, t, true); + add_table_object(col.sub, t+1); + } else { + col.ftype = find_fieldtype(type); + col.sub = NULL; + } + col.dbname = db_column_name(td->columns, td, t); + /* Some schemas repeat, assume they're the same */ + if (find_column(td, col.dbname)) { + tal_free(col.dbname); + } else { + col.jsonname = json_strdup(td->columns, schemas, t); + tal_arr_expand(&td->columns, col); + } + } +} + +/* tok is the JSON schema member for an object */ +static void add_table_object(struct table_desc *td, const jsmntok_t *tok) +{ + const jsmntok_t *t, *properties, *allof, *cond; + size_t i; + + /* This might not exist inside allOf, for example */ + properties = json_get_member(schemas, tok, "properties"); + if (properties) + add_table_properties(td, properties); + + allof = json_get_member(schemas, tok, "allOf"); + if (allof) { + json_for_each_arr(i, t, allof) + add_table_object(td, t); + } + /* We often find interesting things in then and else branches! */ + cond = json_get_member(schemas, tok, "then"); + if (cond) + add_table_object(td, cond); + cond = json_get_member(schemas, tok, "else"); + if (cond) + add_table_object(td, cond); +} + +static void init_tablemap(struct plugin *plugin) +{ + const jsmntok_t *toks, *t; + size_t i; + + strmap_init(&tablemap); + + toks = json_parse_simple(tmpctx, schemas, strlen(schemas)); + json_for_each_obj(i, t, toks) { + struct table_desc *td; + const jsmntok_t *cmd, *items, *type; + + /* First member of properties object is command. */ + cmd = json_get_member(schemas, t+1, "properties") + 1; + + /* We assume it's an object containing an array of objects */ + items = json_get_member(schemas, cmd + 1, "items"); + type = json_get_member(schemas, items, "type"); + assert(json_tok_streq(schemas, type, "object")); + + td = new_table_desc(NULL, t, cmd, false); + tal_steal(plugin, td); + add_table_object(td, items); + + finish_td(plugin, td); + } } #if DEVELOPER diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 2e20dcd8d350..d1790e002972 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3278,7 +3278,9 @@ def test_block_added_notifications(node_factory, bitcoind): def test_sql(node_factory, bitcoind): - l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True) + l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True, + opts={'experimental-offers': None, + 'sqlfilename': 'sql.sqlite3'}) ret = l2.rpc.sql("SELECT * FROM forwards;") assert ret == {'rows': []} @@ -3286,11 +3288,32 @@ def test_sql(node_factory, bitcoind): # This should create a forward through l2 l1.rpc.pay(l3.rpc.invoice(amount_msat=12300, label='inv1', description='description')['bolt11']) + # Very rough checks of other list commands: + ret = l1.rpc.sql("SELECT * FROM htlcs;") + assert len(only_one(ret['rows'])) == 7 + + ret = l3.rpc.sql("SELECT * FROM invoices;") + assert len(only_one(ret['rows'])) == 14 + + ret = l3.rpc.sql("SELECT * FROM nodes;") + assert len(ret['rows']) == 3 + assert len(ret['rows'][0]) == 11 + + ret = l3.rpc.sql("SELECT * FROM peers;") + assert len(only_one(ret['rows'])) == 4 + + l3.rpc.offer(1, 'desc') + ret = l3.rpc.sql("SELECT * FROM offers;") + assert len(only_one(ret['rows'])) == 6 + + ret = l1.rpc.sql("SELECT * FROM sendpays;") + assert len(only_one(ret['rows'])) == 15 + + ret = l3.rpc.sql("SELECT * FROM transactions;") + assert len(only_one(ret['rows'])) == 6 + ret = l2.rpc.sql("SELECT in_htlc_id,out_msat,status,out_htlc_id FROM forwards;") - assert only_one(ret['rows'])[0] == 0 - assert only_one(ret['rows'])[1] == 12300 - assert only_one(ret['rows'])[2] == 'settled' - assert only_one(ret['rows'])[3] == 0 + assert only_one(ret['rows']) == [0, 12300, 'settled', 0] with pytest.raises(RpcError, match='Unauthorized'): l2.rpc.sql("DELETE FROM forwards;") From 1897b3d7f6613f210e2d7cca9a40c40278db8f38 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:17 +1030 Subject: [PATCH 23/36] plugins/sql: make tables for non-object arrays. Signed-off-by: Rusty Russell --- plugins/sql.c | 48 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index f07f68cd86fe..8d3447e4f979 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -71,7 +71,8 @@ static const struct fieldtypemap fieldtypemap[] = { }; struct column { - /* We rename some fields to avoid sql keywords! */ + /* We rename some fields to avoid sql keywords! + * And jsonname is NULL if this is a simple array. */ const char *dbname, *jsonname; enum fieldtype ftype; @@ -366,8 +367,13 @@ static struct command_result *process_json_obj(struct command *cmd, /* This can happen if subobject does not exist in output! */ if (!t) coltok = NULL; - else - coltok = json_get_member(buf, t, col->jsonname); + else { + /* Array of primitives? */ + if (!col->jsonname) + coltok = t; + else + coltok = json_get_member(buf, t, col->jsonname); + } if (!coltok) { if (td->parent) @@ -622,14 +628,6 @@ static bool ignore_column(const struct table_desc *td, const jsmntok_t *t) * ask for it in listpeers, and it's not very useful. */ if (streq(td->name, "peers") && json_tok_streq(schemas, t, "log")) return true; - /* FIXME: peers.netaddr is an array of strings, which we don't handle. */ - if (streq(td->name, "peers") && json_tok_streq(schemas, t, "netaddr")) - return true; - /* FIXME: peers.channels.features is an array of strings, which we don't handle. */ - if (streq(td->name, "peers_channels") && json_tok_streq(schemas, t, "features")) - return true; - if (streq(td->name, "peers_channels") && json_tok_streq(schemas, t, "status")) - return true; return false; } @@ -777,6 +775,27 @@ static bool find_column(const struct table_desc *td, /* Recursion */ static void add_table_object(struct table_desc *td, const jsmntok_t *tok); +/* Simple case for arrays of a simple type. */ +static void add_table_singleton(struct table_desc *td, + const jsmntok_t *name, + const jsmntok_t *tok) +{ + struct column col; + const jsmntok_t *type; + + /* FIXME: We would need to return false here and delete table! */ + assert(!ignore_column(td, tok)); + type = json_get_member(schemas, tok, "type"); + + col.ftype = find_fieldtype(type); + col.sub = NULL; + /* We name column after the JSON parent field; but jsonname is NULL so we + * know to expect an array not a member. */ + col.dbname = db_column_name(td->columns, td, name); + col.jsonname = NULL; + tal_arr_expand(&td->columns, col); +} + static void add_table_properties(struct table_desc *td, const jsmntok_t *properties) { @@ -799,10 +818,13 @@ static void add_table_properties(struct table_desc *td, items = json_get_member(schemas, t+1, "items"); type = json_get_member(schemas, items, "type"); - assert(json_tok_streq(schemas, type, "object")); col.sub = new_table_desc(td, t, t, false); - add_table_object(col.sub, items); + /* Array of primitives? Treat as single-entry obj */ + if (!json_tok_streq(schemas, type, "object")) + add_table_singleton(col.sub, t, items); + else + add_table_object(col.sub, items); } else if (json_tok_streq(schemas, type, "object")) { col.sub = new_table_desc(td, t, t, true); add_table_object(col.sub, t+1); From 1b58d3722fd9d2ac6ffef4530439a1db9b7eeb6b Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:17 +1030 Subject: [PATCH 24/36] plugins/sql: add listpeerchannels support. Signed-off-by: Rusty Russell --- plugins/Makefile | 2 +- tests/test_plugin.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/Makefile b/plugins/Makefile index b1a4e8363ec3..6d92bb880dde 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -205,7 +205,7 @@ plugins/fetchinvoice: $(PLUGIN_FETCHINVOICE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_CO plugins/funder: bitcoin/psbt.o common/psbt_open.o $(PLUGIN_FUNDER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) # This covers all the low-level list RPCs which return simple arrays -SQL_LISTRPCS := listchannels listforwards listhtlcs listinvoices listnodes listoffers listpeers listtransactions listsendpays +SQL_LISTRPCS := listchannels listforwards listhtlcs listinvoices listnodes listoffers listpeers listpeerchannels listtransactions listsendpays SQL_LISTRPCS_SCHEMAS := $(foreach l,$(SQL_LISTRPCS),doc/schemas/$l.schema.json) # We squeeze: # descriptions (we don't need) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index d1790e002972..b995b3f2edcf 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3302,6 +3302,9 @@ def test_sql(node_factory, bitcoind): ret = l3.rpc.sql("SELECT * FROM peers;") assert len(only_one(ret['rows'])) == 4 + ret = l3.rpc.sql("SELECT * FROM peerchannels;") + assert len(only_one(ret['rows'])) == 57 + l3.rpc.offer(1, 'desc') ret = l3.rpc.sql("SELECT * FROM offers;") assert len(only_one(ret['rows'])) == 6 From b0c2560901022d175f93a29032d03788edda07ac Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:17 +1030 Subject: [PATCH 25/36] pytest: perform more thorough testing. Painfully created by hand from the source. Signed-off-by: Rusty Russell --- tests/test_plugin.py | 498 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 470 insertions(+), 28 deletions(-) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index b995b3f2edcf..748f61e5dd56 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3277,10 +3277,13 @@ def test_block_added_notifications(node_factory, bitcoind): assert len(ret) == 3 and ret[1] == next_l2_base + 1 and ret[2] == next_l2_base + 2 +@pytest.mark.openchannel('v2') +@pytest.mark.developer("wants dev-announce-localhost so we see listnodes.addresses") def test_sql(node_factory, bitcoind): l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True, opts={'experimental-offers': None, - 'sqlfilename': 'sql.sqlite3'}) + 'sqlfilename': 'sql.sqlite3', + 'dev-allow-localhost': None}) ret = l2.rpc.sql("SELECT * FROM forwards;") assert ret == {'rows': []} @@ -3288,34 +3291,473 @@ def test_sql(node_factory, bitcoind): # This should create a forward through l2 l1.rpc.pay(l3.rpc.invoice(amount_msat=12300, label='inv1', description='description')['bolt11']) - # Very rough checks of other list commands: - ret = l1.rpc.sql("SELECT * FROM htlcs;") - assert len(only_one(ret['rows'])) == 7 - - ret = l3.rpc.sql("SELECT * FROM invoices;") - assert len(only_one(ret['rows'])) == 14 - - ret = l3.rpc.sql("SELECT * FROM nodes;") - assert len(ret['rows']) == 3 - assert len(ret['rows'][0]) == 11 - - ret = l3.rpc.sql("SELECT * FROM peers;") - assert len(only_one(ret['rows'])) == 4 - - ret = l3.rpc.sql("SELECT * FROM peerchannels;") - assert len(only_one(ret['rows'])) == 57 - - l3.rpc.offer(1, 'desc') - ret = l3.rpc.sql("SELECT * FROM offers;") - assert len(only_one(ret['rows'])) == 6 - - ret = l1.rpc.sql("SELECT * FROM sendpays;") - assert len(only_one(ret['rows'])) == 15 - - ret = l3.rpc.sql("SELECT * FROM transactions;") - assert len(only_one(ret['rows'])) == 6 + expected_schemas = { + 'channels': { + 'columns': [{'name': 'source', + 'type': 'pubkey'}, + {'name': 'destination', + 'type': 'pubkey'}, + {'name': 'short_channel_id', + 'type': 'short_channel_id'}, + {'name': 'direction', + 'type': 'u32'}, + {'name': 'public', + 'type': 'boolean'}, + {'name': 'amount_msat', + 'type': 'msat'}, + {'name': 'message_flags', + 'type': 'u8'}, + {'name': 'channel_flags', + 'type': 'u8'}, + {'name': 'active', + 'type': 'boolean'}, + {'name': 'last_update', + 'type': 'u32'}, + {'name': 'base_fee_millisatoshi', + 'type': 'u32'}, + {'name': 'fee_per_millionth', + 'type': 'u32'}, + {'name': 'delay', + 'type': 'u32'}, + {'name': 'htlc_minimum_msat', + 'type': 'msat'}, + {'name': 'htlc_maximum_msat', + 'type': 'msat'}, + {'name': 'features', + 'type': 'hex'}]}, + 'nodes': { + 'columns': [{'name': 'nodeid', + 'type': 'pubkey'}, + {'name': 'last_timestamp', + 'type': 'u32'}, + {'name': 'alias', + 'type': 'string'}, + {'name': 'color', + 'type': 'hex'}, + {'name': 'features', + 'type': 'hex'}, + {'name': 'option_will_fund_lease_fee_base_msat', + 'type': 'msat'}, + {'name': 'option_will_fund_lease_fee_basis', + 'type': 'u32'}, + {'name': 'option_will_fund_funding_weight', + 'type': 'u32'}, + {'name': 'option_will_fund_channel_fee_max_base_msat', + 'type': 'msat'}, + {'name': 'option_will_fund_channel_fee_max_proportional_thousandths', + 'type': 'u32'}, + {'name': 'option_will_fund_compact_lease', + 'type': 'hex'}, + ]}, + 'nodes_addresses': { + 'columns': [{'name': 'row', + 'type': 'u64'}, + {'name': 'arrindex', + 'type': 'u64'}, + {'name': 'type', + 'type': 'string'}, + {'name': 'port', + 'type': 'u16'}, + {'name': 'address', + 'type': 'string'}]}, + 'forwards': { + 'columns': [{'name': 'in_channel', + 'type': 'short_channel_id'}, + {'name': 'in_htlc_id', + 'type': 'u64'}, + {'name': 'in_msat', + 'type': 'msat'}, + {'name': 'status', + 'type': 'string'}, + {'name': 'received_time', + 'type': 'number'}, + {'name': 'out_channel', + 'type': 'short_channel_id'}, + {'name': 'out_htlc_id', + 'type': 'u64'}, + {'name': 'style', + 'type': 'string'}, + {'name': 'fee_msat', + 'type': 'msat'}, + {'name': 'out_msat', + 'type': 'msat'}, + {'name': 'resolved_time', + 'type': 'number'}, + {'name': 'failcode', + 'type': 'u32'}, + {'name': 'failreason', + 'type': 'string'}]}, + 'htlcs': { + 'columns': [{'name': 'short_channel_id', + 'type': 'short_channel_id'}, + {'name': 'id', + 'type': 'u64'}, + {'name': 'expiry', + 'type': 'u32'}, + {'name': 'amount_msat', + 'type': 'msat'}, + {'name': 'direction', + 'type': 'string'}, + {'name': 'payment_hash', + 'type': 'hash'}, + {'name': 'state', + 'type': 'string'}]}, + 'invoices': { + 'columns': [{'name': 'label', + 'type': 'string'}, + {'name': 'description', + 'type': 'string'}, + {'name': 'payment_hash', + 'type': 'hash'}, + {'name': 'status', + 'type': 'string'}, + {'name': 'expires_at', + 'type': 'u64'}, + {'name': 'amount_msat', + 'type': 'msat'}, + {'name': 'bolt11', + 'type': 'string'}, + {'name': 'bolt12', + 'type': 'string'}, + {'name': 'local_offer_id', + 'type': 'hex'}, + {'name': 'invreq_payer_note', + 'type': 'string'}, + {'name': 'pay_index', + 'type': 'u64'}, + {'name': 'amount_received_msat', + 'type': 'msat'}, + {'name': 'paid_at', + 'type': 'u64'}, + {'name': 'payment_preimage', + 'type': 'secret'}]}, + 'offers': { + 'columns': [{'name': 'offer_id', + 'type': 'hex'}, + {'name': 'active', + 'type': 'boolean'}, + {'name': 'single_use', + 'type': 'boolean'}, + {'name': 'bolt12', + 'type': 'string'}, + {'name': 'used', + 'type': 'boolean'}, + {'name': 'label', + 'type': 'string'}]}, + 'peers': { + 'columns': [{'name': 'id', + 'type': 'pubkey'}, + {'name': 'connected', + 'type': 'boolean'}, + {'name': 'remote_addr', + 'type': 'string'}, + {'name': 'features', + 'type': 'hex'}]}, + 'peers_netaddr': { + 'columns': [{'name': 'row', + 'type': 'u64'}, + {'name': 'arrindex', + 'type': 'u64'}, + {'name': 'netaddr', + 'type': 'string'}]}, + 'sendpays': { + 'columns': [{'name': 'id', + 'type': 'u64'}, + {'name': 'groupid', + 'type': 'u64'}, + {'name': 'partid', + 'type': 'u64'}, + {'name': 'payment_hash', + 'type': 'hash'}, + {'name': 'status', + 'type': 'string'}, + {'name': 'amount_msat', + 'type': 'msat'}, + {'name': 'destination', + 'type': 'pubkey'}, + {'name': 'created_at', + 'type': 'u64'}, + {'name': 'amount_sent_msat', + 'type': 'msat'}, + {'name': 'label', + 'type': 'string'}, + {'name': 'bolt11', + 'type': 'string'}, + {'name': 'description', + 'type': 'string'}, + {'name': 'bolt12', + 'type': 'string'}, + {'name': 'payment_preimage', + 'type': 'secret'}, + {'name': 'erroronion', + 'type': 'hex'}]}, + 'peerchannels': { + 'columns': [{'name': 'peer_id', + 'type': 'pubkey'}, + {'name': 'peer_connected', + 'type': 'boolean'}, + {'name': 'state', + 'type': 'string'}, + {'name': 'scratch_txid', + 'type': 'txid'}, + {'name': 'feerate_perkw', + 'type': 'u32'}, + {'name': 'feerate_perkb', + 'type': 'u32'}, + {'name': 'owner', + 'type': 'string'}, + {'name': 'short_channel_id', + 'type': 'short_channel_id'}, + {'name': 'channel_id', + 'type': 'hash'}, + {'name': 'funding_txid', + 'type': 'txid'}, + {'name': 'funding_outnum', + 'type': 'u32'}, + {'name': 'initial_feerate', + 'type': 'string'}, + {'name': 'last_feerate', + 'type': 'string'}, + {'name': 'next_feerate', + 'type': 'string'}, + {'name': 'next_fee_step', + 'type': 'u32'}, + {'name': 'close_to', + 'type': 'hex'}, + {'name': 'private', + 'type': 'boolean'}, + {'name': 'opener', + 'type': 'string'}, + {'name': 'closer', + 'type': 'string'}, + {'name': 'funding_local_msat', + 'type': 'msat'}, + {'name': 'funding_remote_msat', + 'type': 'msat'}, + {'name': 'funding_pushed_msat', + 'type': 'msat'}, + {'name': 'funding_local_funds_msat', + 'type': 'msat'}, + {'name': 'funding_remote_funds_msat', + 'type': 'msat'}, + {'name': 'funding_fee_paid_msat', + 'type': 'msat'}, + {'name': 'funding_fee_rcvd_msat', + 'type': 'msat'}, + {'name': 'to_us_msat', + 'type': 'msat'}, + {'name': 'min_to_us_msat', + 'type': 'msat'}, + {'name': 'max_to_us_msat', + 'type': 'msat'}, + {'name': 'total_msat', + 'type': 'msat'}, + {'name': 'fee_base_msat', + 'type': 'msat'}, + {'name': 'fee_proportional_millionths', + 'type': 'u32'}, + {'name': 'dust_limit_msat', + 'type': 'msat'}, + {'name': 'max_total_htlc_in_msat', + 'type': 'msat'}, + {'name': 'their_reserve_msat', + 'type': 'msat'}, + {'name': 'our_reserve_msat', + 'type': 'msat'}, + {'name': 'spendable_msat', + 'type': 'msat'}, + {'name': 'receivable_msat', + 'type': 'msat'}, + {'name': 'minimum_htlc_in_msat', + 'type': 'msat'}, + {'name': 'minimum_htlc_out_msat', + 'type': 'msat'}, + {'name': 'maximum_htlc_out_msat', + 'type': 'msat'}, + {'name': 'their_to_self_delay', + 'type': 'u32'}, + {'name': 'our_to_self_delay', + 'type': 'u32'}, + {'name': 'max_accepted_htlcs', + 'type': 'u32'}, + {'name': 'alias_local', + 'type': 'short_channel_id'}, + {'name': 'alias_remote', + 'type': 'short_channel_id'}, + {'name': 'in_payments_offered', + 'type': 'u64'}, + {'name': 'in_offered_msat', + 'type': 'msat'}, + {'name': 'in_payments_fulfilled', + 'type': 'u64'}, + {'name': 'in_fulfilled_msat', + 'type': 'msat'}, + {'name': 'out_payments_offered', + 'type': 'u64'}, + {'name': 'out_offered_msat', + 'type': 'msat'}, + {'name': 'out_payments_fulfilled', + 'type': 'u64'}, + {'name': 'out_fulfilled_msat', + 'type': 'msat'}, + {'name': 'close_to_addr', + 'type': 'string'}, + {'name': 'last_tx_fee_msat', + 'type': 'msat'}, + {'name': 'direction', + 'type': 'u32'}]}, + 'peerchannels_features': { + 'columns': [{'name': 'row', + 'type': 'u64'}, + {'name': 'arrindex', + 'type': 'u64'}, + {'name': 'features', + 'type': 'string'}]}, + 'peerchannels_htlcs': { + 'columns': [{'name': 'row', + 'type': 'u64'}, + {'name': 'arrindex', + 'type': 'u64'}, + {'name': 'direction', + 'type': 'string'}, + {'name': 'id', + 'type': 'u64'}, + {'name': 'amount_msat', + 'type': 'msat'}, + {'name': 'expiry', + 'type': 'u32'}, + {'name': 'payment_hash', + 'type': 'hash'}, + {'name': 'local_trimmed', + 'type': 'boolean'}, + {'name': 'status', + 'type': 'string'}, + {'name': 'state', + 'type': 'string'}]}, + 'peerchannels_inflight': { + 'columns': [{'name': 'row', + 'type': 'u64'}, + {'name': 'arrindex', + 'type': 'u64'}, + {'name': 'funding_txid', + 'type': 'txid'}, + {'name': 'funding_outnum', + 'type': 'u32'}, + {'name': 'feerate', + 'type': 'string'}, + {'name': 'total_funding_msat', + 'type': 'msat'}, + {'name': 'our_funding_msat', + 'type': 'msat'}, + {'name': 'scratch_txid', + 'type': 'txid'}]}, + 'peerchannels_status': { + 'columns': [{'name': 'row', + 'type': 'u64'}, + {'name': 'arrindex', + 'type': 'u64'}, + {'name': 'status', + 'type': 'string'}]}, + 'peerchannels_state_changes': { + 'columns': [{'name': 'row', + 'type': 'u64'}, + {'name': 'arrindex', + 'type': 'u64'}, + {'name': 'timestamp', + 'type': 'string'}, + {'name': 'old_state', + 'type': 'string'}, + {'name': 'new_state', + 'type': 'string'}, + {'name': 'cause', + 'type': 'string'}, + {'name': 'message', + 'type': 'string'}]}, + 'transactions': { + 'columns': [{'name': 'hash', + 'type': 'txid'}, + {'name': 'rawtx', + 'type': 'hex'}, + {'name': 'blockheight', + 'type': 'u32'}, + {'name': 'txindex', + 'type': 'u32'}, + {'name': 'locktime', + 'type': 'u32'}, + {'name': 'version', + 'type': 'u32'}]}, + 'transactions_inputs': { + 'columns': [{'name': 'row', + 'type': 'u64'}, + {'name': 'arrindex', + 'type': 'u64'}, + {'name': 'txid', + 'type': 'hex'}, + {'name': 'idx', + 'type': 'u32'}, + {'name': 'sequence', + 'type': 'u32'}, + {'name': 'type', + 'type': 'string'}, + {'name': 'channel', + 'type': 'short_channel_id'}]}, + 'transactions_outputs': { + 'columns': [{'name': 'row', + 'type': 'u64'}, + {'name': 'arrindex', + 'type': 'u64'}, + {'name': 'idx', + 'type': 'u32'}, + {'name': 'amount_msat', + 'type': 'msat'}, + {'name': 'scriptPubKey', + 'type': 'hex'}, + {'name': 'type', + 'type': 'string'}, + {'name': 'channel', + 'type': 'short_channel_id'}]}} + + # Very rough checks of other list commands (make sure l2 has one of each) + l2.rpc.offer(1, 'desc') + l2.rpc.invoice(1, 'label', 'desc') + l2.rpc.pay(l3.rpc.invoice(amount_msat=12300, label='inv2', description='description')['bolt11']) + + # And I need at least one HTLC in-flight so listpeers.channels.htlcs isn't empty: + l3.rpc.plugin_start(os.path.join(os.getcwd(), 'tests/plugins/hold_invoice.py'), + holdtime=10000) + inv = l3.rpc.invoice(amount_msat=12300, label='inv3', description='description') + route = l1.rpc.getroute(l3.info['id'], 12300, 1)['route'] + l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret']) + # And an in-flight channel open... + l2.openchannel(l3, confirm=False, wait_for_announce=False) - ret = l2.rpc.sql("SELECT in_htlc_id,out_msat,status,out_htlc_id FROM forwards;") + for table, schema in expected_schemas.items(): + ret = l2.rpc.sql("SELECT * FROM {};".format(table)) + assert len(ret['rows'][0]) == len(schema['columns']) + + for col in schema['columns']: + val = only_one(l2.rpc.sql("SELECT {} FROM {};".format(col['name'], table))['rows'][0]) + # Could be null + if val is None: + continue + if col['type'] == "hex": + bytes.fromhex(val) + elif col['type'] in ("hash", "secret", "txid"): + assert len(bytes.fromhex(val)) == 32 + elif col['type'] == "pubkey": + assert len(bytes.fromhex(val)) == 33 + elif col['type'] in ("msat", "integer", "u64", "u32", "u16", "u8", "boolean"): + int(val) + elif col['type'] == "number": + float(val) + elif col['type'] == "string": + val += "" + elif col['type'] == "short_channel_id": + assert len(val.split('x')) == 3 + else: + assert False + + ret = l2.rpc.sql("SELECT in_htlc_id,out_msat,status,out_htlc_id FROM forwards WHERE in_htlc_id = 0;") assert only_one(ret['rows']) == [0, 12300, 'settled', 0] with pytest.raises(RpcError, match='Unauthorized'): From 4bf9d21ac277896f05bf60c41df38b3152cf50d1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:17 +1030 Subject: [PATCH 26/36] plugins/sql: include the obvious indexes. Signed-off-by: Rusty Russell --- plugins/sql.c | 72 ++++++++++++++++++++++++++++++++++++++++++-- tests/test_plugin.py | 10 ++++++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index 8d3447e4f979..f2ef3a8b8787 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -23,7 +23,6 @@ static const char schemas[] = * 6. test on mainnet. * 7. Some cool query for documentation. * 8. time_msec fields. - * 9. Primary key in schema? * 10. Pagination API */ enum fieldtype { @@ -109,6 +108,56 @@ static size_t max_dbmem = 500000000; static struct sqlite3 *db; static const char *dbfilename; +/* It was tempting to put these in the schema, but they're really + * just for our usage. Though that would allow us to autogen the + * documentation, too. */ +struct index { + const char *tablename; + const char *fields[2]; +}; +static const struct index indices[] = { + { + "channels", + { "short_channel_id", NULL }, + }, + { + "forwards", + { "in_channel", "in_htlc_id" }, + }, + { + "htlcs", + { "short_channel_id", "id" }, + }, + { + "invoices", + { "payment_hash", NULL }, + }, + { + "nodes", + { "nodeid", NULL }, + }, + { + "offers", + { "offer_id", NULL }, + }, + { + "peers", + { "id", NULL }, + }, + { + "peerchannels", + { "peer_id", NULL }, + }, + { + "sendpays", + { "payment_hash", NULL }, + }, + { + "transactions", + { "hash", NULL }, + }, +}; + static enum fieldtype find_fieldtype(const jsmntok_t *name) { for (size_t i = 0; i < ARRAY_SIZE(fieldtypemap); i++) { @@ -643,7 +692,6 @@ static void finish_td(struct plugin *plugin, struct table_desc *td) if (td->is_subobject) return; - /* FIXME: Primary key from schema? */ create_stmt = tal_fmt(tmpctx, "CREATE TABLE %s (", td->name); td->update_stmt = tal_fmt(td, "INSERT INTO %s VALUES (", td->name); @@ -896,6 +944,25 @@ static void init_tablemap(struct plugin *plugin) } } +static void init_indices(struct plugin *plugin) +{ + for (size_t i = 0; i < ARRAY_SIZE(indices); i++) { + char *errmsg, *cmd; + int err; + + cmd = tal_fmt(tmpctx, "CREATE INDEX %s_%zu_idx ON %s (%s", + indices[i].tablename, i, + indices[i].tablename, + indices[i].fields[0]); + if (indices[i].fields[1]) + tal_append_fmt(&cmd, ", %s", indices[i].fields[1]); + tal_append_fmt(&cmd, ");"); + err = sqlite3_exec(db, cmd, NULL, NULL, &errmsg); + if (err != SQLITE_OK) + plugin_err(plugin, "Failed '%s': %s", cmd, errmsg); + } +} + #if DEVELOPER static void memleak_mark_tablemap(struct plugin *p, struct htable *memtable) { @@ -909,6 +976,7 @@ static const char *init(struct plugin *plugin, { db = sqlite_setup(plugin); init_tablemap(plugin); + init_indices(plugin); #if DEVELOPER plugin_set_memleak_handler(plugin, memleak_mark_tablemap); diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 748f61e5dd56..75a460e4a6b5 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3293,6 +3293,7 @@ def test_sql(node_factory, bitcoind): expected_schemas = { 'channels': { + 'indices': [['short_channel_id']], 'columns': [{'name': 'source', 'type': 'pubkey'}, {'name': 'destination', @@ -3326,6 +3327,7 @@ def test_sql(node_factory, bitcoind): {'name': 'features', 'type': 'hex'}]}, 'nodes': { + 'indices': [['nodeid']], 'columns': [{'name': 'nodeid', 'type': 'pubkey'}, {'name': 'last_timestamp', @@ -3361,6 +3363,7 @@ def test_sql(node_factory, bitcoind): {'name': 'address', 'type': 'string'}]}, 'forwards': { + 'indices': [['in_channel', 'in_htlc_id']], 'columns': [{'name': 'in_channel', 'type': 'short_channel_id'}, {'name': 'in_htlc_id', @@ -3388,6 +3391,7 @@ def test_sql(node_factory, bitcoind): {'name': 'failreason', 'type': 'string'}]}, 'htlcs': { + 'indices': [['short_channel_id', 'id']], 'columns': [{'name': 'short_channel_id', 'type': 'short_channel_id'}, {'name': 'id', @@ -3403,6 +3407,7 @@ def test_sql(node_factory, bitcoind): {'name': 'state', 'type': 'string'}]}, 'invoices': { + 'indices': [['payment_hash']], 'columns': [{'name': 'label', 'type': 'string'}, {'name': 'description', @@ -3432,6 +3437,7 @@ def test_sql(node_factory, bitcoind): {'name': 'payment_preimage', 'type': 'secret'}]}, 'offers': { + 'indices': [['offer_id']], 'columns': [{'name': 'offer_id', 'type': 'hex'}, {'name': 'active', @@ -3445,6 +3451,7 @@ def test_sql(node_factory, bitcoind): {'name': 'label', 'type': 'string'}]}, 'peers': { + 'indices': [['id']], 'columns': [{'name': 'id', 'type': 'pubkey'}, {'name': 'connected', @@ -3461,6 +3468,7 @@ def test_sql(node_factory, bitcoind): {'name': 'netaddr', 'type': 'string'}]}, 'sendpays': { + 'indices': [['payment_hash']], 'columns': [{'name': 'id', 'type': 'u64'}, {'name': 'groupid', @@ -3492,6 +3500,7 @@ def test_sql(node_factory, bitcoind): {'name': 'erroronion', 'type': 'hex'}]}, 'peerchannels': { + 'indices': [['peer_id']], 'columns': [{'name': 'peer_id', 'type': 'pubkey'}, {'name': 'peer_connected', @@ -3674,6 +3683,7 @@ def test_sql(node_factory, bitcoind): {'name': 'message', 'type': 'string'}]}, 'transactions': { + 'indices': [['hash']], 'columns': [{'name': 'hash', 'type': 'txid'}, {'name': 'rawtx', From 0a6cd62b09a8ea7628b3bef06d467e74eb8b67a1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:17 +1030 Subject: [PATCH 27/36] plugins/sql: refresh listnodes and listchannels by monitoring the gossip_store. It's quite a lot of code, but these are the most expensive commands we do so it's worth it. Signed-off-by: Rusty Russell --- plugins/Makefile | 2 +- plugins/sql.c | 296 ++++++++++++++++++++++++++++++++++++++++++- tests/test_plugin.py | 17 ++- 3 files changed, 312 insertions(+), 3 deletions(-) diff --git a/plugins/Makefile b/plugins/Makefile index 6d92bb880dde..fad17d248b74 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -218,7 +218,7 @@ plugins/sql-schema_gen.h: plugins/Makefile $(SQL_LISTRPCS_SCHEMAS) @$(call VERBOSE,GEN $@, (SEP=""; echo -n '"{'; for f in $(SQL_LISTRPCS); do echo -n "$$SEP\\\"$$f\\\":"; sed -e s/\"description\":\ *\".\*\"/\"\":\"\"/ -e s/\".*\":\ *{}/\"\":{}/ -e s/\"/\\\\\"/g < doc/schemas/$$f.schema.json | tr -d ' \n'; SEP=","; done; echo '}"') > $@) plugins/sql.o: plugins/sql-schema_gen.h -plugins/sql: $(PLUGIN_SQL_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) +plugins/sql: $(PLUGIN_SQL_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) common/gossip_store.o gossipd/gossip_store_wiregen.o # Generated from PLUGINS definition in plugins/Makefile ALL_C_HEADERS += plugins/list_of_builtin_plugins_gen.h diff --git a/plugins/sql.c b/plugins/sql.c index f2ef3a8b8787..00d5a98fea09 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -4,11 +4,19 @@ #include #include #include +#include #include #include #include +#include +#include +#include +#include #include #include +#include +#include +#include /* Minimized schemas. C23 #embed, Where Art Thou? */ static const char schemas[] = @@ -107,6 +115,8 @@ static STRMAP(struct table_desc *) tablemap; static size_t max_dbmem = 500000000; static struct sqlite3 *db; static const char *dbfilename; +static int gosstore_fd = -1; +static size_t gosstore_nodes_off = 0, gosstore_channels_off = 0; /* It was tempting to put these in the schema, but they're really * just for our usage. Though that would allow us to autogen the @@ -622,6 +632,285 @@ static struct command_result *default_refresh(struct command *cmd, return send_outreq(cmd->plugin, req); } +static bool extract_scid(int gosstore_fd, size_t off, u16 type, + struct short_channel_id *scid) +{ + be64 raw; + + /* BOLT #7: + * 1. type: 258 (`channel_update`) + * 2. data: + * * [`signature`:`signature`] + * * [`chain_hash`:`chain_hash`] + * * [`short_channel_id`:`short_channel_id`] + */ + /* Note that first two bytes are message type */ + const size_t update_scid_off = 2 + (64 + 32); + + off += sizeof(struct gossip_hdr); + /* For delete_chan scid immediately follows type */ + if (type == WIRE_GOSSIP_STORE_DELETE_CHAN) + off += 2; + else if (type == WIRE_GOSSIP_STORE_PRIVATE_UPDATE) + /* Prepend header */ + off += 2 + 2 + update_scid_off; + else if (type == WIRE_CHANNEL_UPDATE) + off += update_scid_off; + else + abort(); + + if (pread(gosstore_fd, &raw, sizeof(raw), off) != sizeof(raw)) + return false; + scid->u64 = be64_to_cpu(raw); + return true; +} + +/* Note: this deletes up to two rows, one for each direction. */ +static void delete_channel_from_db(struct command *cmd, + struct short_channel_id scid) +{ + int err; + char *errmsg; + + err = sqlite3_exec(db, + tal_fmt(tmpctx, + "DELETE FROM channels" + " WHERE short_channel_id = '%s'", + short_channel_id_to_str(tmpctx, &scid)), + NULL, NULL, &errmsg); + if (err != SQLITE_OK) + plugin_err(cmd->plugin, "Could not delete from channels: %s", + errmsg); +} + +static struct command_result *channels_refresh(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq); + +static struct command_result *listchannels_one_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct db_query *dbq) +{ + const struct table_desc *td = dbq->tables[0]; + struct command_result *ret; + + ret = process_json_result(cmd, buf, result, td); + if (ret) + return ret; + + /* Continue to refresh more channels */ + return channels_refresh(cmd, td, dbq); +} + +static struct command_result *channels_refresh(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq) +{ + struct out_req *req; + size_t msglen; + u16 type, flags; + + if (gosstore_fd == -1) { + gosstore_fd = open("gossip_store", O_RDONLY); + if (gosstore_fd == -1) + plugin_err(cmd->plugin, "Could not open gossip_store: %s", + strerror(errno)); + } + + /* First time, set off to end and load from scratch */ + if (gosstore_channels_off == 0) { + gosstore_channels_off = find_gossip_store_end(gosstore_fd, 1); + return default_refresh(cmd, td, dbq); + } + + plugin_log(cmd->plugin, LOG_DBG, "Refreshing channels @%zu...", + gosstore_channels_off); + + /* OK, try catching up! */ + while (gossip_store_readhdr(gosstore_fd, gosstore_channels_off, + &msglen, NULL, &flags, &type)) { + struct short_channel_id scid; + size_t off = gosstore_channels_off; + + gosstore_channels_off += sizeof(struct gossip_hdr) + msglen; + + if (flags & GOSSIP_STORE_DELETED_BIT) + continue; + + if (type == WIRE_GOSSIP_STORE_ENDED) { + /* Force a reopen */ + gosstore_channels_off = gosstore_nodes_off = 0; + close(gosstore_fd); + gosstore_fd = -1; + return channels_refresh(cmd, td, dbq); + } + + /* If we see a channel_announcement, we don't care until we + * see the channel_update */ + if (type == WIRE_CHANNEL_UPDATE + || type == WIRE_GOSSIP_STORE_PRIVATE_UPDATE) { + /* This can fail if entry not fully written yet. */ + if (!extract_scid(gosstore_fd, off, type, &scid)) { + gosstore_channels_off = off; + break; + } + + plugin_log(cmd->plugin, LOG_DBG, "Refreshing channel: %s", + type_to_string(tmpctx, struct short_channel_id, &scid)); + /* FIXME: sqlite 3.24.0 (2018-06-04) added UPSERT, but + * we don't require it. */ + delete_channel_from_db(cmd, scid); + req = jsonrpc_request_start(cmd->plugin, cmd, "listchannels", + listchannels_one_done, + forward_error, + dbq); + json_add_short_channel_id(req->js, "short_channel_id", &scid); + return send_outreq(cmd->plugin, req); + } else if (type == WIRE_GOSSIP_STORE_DELETE_CHAN) { + /* This can fail if entry not fully written yet. */ + if (!extract_scid(gosstore_fd, off, type, &scid)) { + gosstore_channels_off = off; + break; + } + plugin_log(cmd->plugin, LOG_DBG, "Deleting channel: %s", + type_to_string(tmpctx, struct short_channel_id, &scid)); + delete_channel_from_db(cmd, scid); + } + } + + return one_refresh_done(cmd, dbq); +} + +static struct command_result *nodes_refresh(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq); + +static struct command_result *listnodes_one_done(struct command *cmd, + const char *buf, + const jsmntok_t *result, + struct db_query *dbq) +{ + const struct table_desc *td = dbq->tables[0]; + struct command_result *ret; + + ret = process_json_result(cmd, buf, result, td); + if (ret) + return ret; + + /* Continue to refresh more nodes */ + return nodes_refresh(cmd, td, dbq); +} + +static void delete_node_from_db(struct command *cmd, + const struct node_id *id) +{ + int err; + char *errmsg; + + err = sqlite3_exec(db, + tal_fmt(tmpctx, + "DELETE FROM nodes" + " WHERE nodeid = %s", + node_id_to_hexstr(tmpctx, id)), + NULL, NULL, &errmsg); + if (err != SQLITE_OK) + plugin_err(cmd->plugin, "Could not delete from nodes: %s", + errmsg); +} + +static bool extract_node_id(int gosstore_fd, size_t off, u16 type, + struct node_id *id) +{ + /* BOLT #7: + * 1. type: 257 (`node_announcement`) + * 2. data: + * * [`signature`:`signature`] + * * [`u16`:`flen`] + * * [`flen*byte`:`features`] + * * [`u32`:`timestamp`] + * * [`point`:`node_id`] + */ + const size_t feature_len_off = 2 + 64; + be16 flen; + size_t node_id_off; + + off += sizeof(struct gossip_hdr); + + if (pread(gosstore_fd, &flen, sizeof(flen), off + feature_len_off) + != sizeof(flen)) + return false; + + node_id_off = off + feature_len_off + 2 + flen + 4; + if (pread(gosstore_fd, id, sizeof(*id), node_id_off) != sizeof(*id)) + return false; + + return true; +} + +static struct command_result *nodes_refresh(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq) +{ + struct out_req *req; + size_t msglen; + u16 type, flags; + + if (gosstore_fd == -1) { + gosstore_fd = open("gossip_store", O_RDONLY); + if (gosstore_fd == -1) + plugin_err(cmd->plugin, "Could not open gossip_store: %s", + strerror(errno)); + } + + /* First time, set off to end and load from scratch */ + if (gosstore_nodes_off == 0) { + gosstore_nodes_off = find_gossip_store_end(gosstore_fd, 1); + return default_refresh(cmd, td, dbq); + } + + /* OK, try catching up! */ + while (gossip_store_readhdr(gosstore_fd, gosstore_nodes_off, + &msglen, NULL, &flags, &type)) { + struct node_id id; + size_t off = gosstore_nodes_off; + + gosstore_nodes_off += sizeof(struct gossip_hdr) + msglen; + + if (flags & GOSSIP_STORE_DELETED_BIT) + continue; + + if (type == WIRE_GOSSIP_STORE_ENDED) { + /* Force a reopen */ + gosstore_nodes_off = gosstore_channels_off = 0; + close(gosstore_fd); + gosstore_fd = -1; + return nodes_refresh(cmd, td, dbq); + } + + if (type == WIRE_NODE_ANNOUNCEMENT) { + /* This can fail if entry not fully written yet. */ + if (!extract_node_id(gosstore_fd, off, type, &id)) { + gosstore_nodes_off = off; + break; + } + + /* FIXME: sqlite 3.24.0 (2018-06-04) added UPSERT, but + * we don't require it. */ + delete_node_from_db(cmd, &id); + req = jsonrpc_request_start(cmd->plugin, cmd, "listnodes", + listnodes_one_done, + forward_error, + dbq); + json_add_node_id(req->js, "id", &id); + return send_outreq(cmd->plugin, req); + } + /* FIXME: Add WIRE_GOSSIP_STORE_DELETE_NODE marker! */ + } + + return one_refresh_done(cmd, dbq); +} + static struct command_result *refresh_tables(struct command *cmd, struct db_query *dbq) { @@ -806,7 +1095,12 @@ static struct table_desc *new_table_desc(struct table_desc *parent, td->is_subobject = is_subobject; td->arrname = json_strdup(td, schemas, arrname); td->columns = tal_arr(td, struct column, 0); - td->refresh = default_refresh; + if (streq(td->name, "channels")) + td->refresh = channels_refresh; + else if (streq(td->name, "nodes")) + td->refresh = nodes_refresh; + else + td->refresh = default_refresh; return td; } diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 75a460e4a6b5..484db96a0f1b 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3734,7 +3734,7 @@ def test_sql(node_factory, bitcoind): # And I need at least one HTLC in-flight so listpeers.channels.htlcs isn't empty: l3.rpc.plugin_start(os.path.join(os.getcwd(), 'tests/plugins/hold_invoice.py'), - holdtime=10000) + holdtime=TIMEOUT * 2) inv = l3.rpc.invoice(amount_msat=12300, label='inv3', description='description') route = l1.rpc.getroute(l3.info['id'], 12300, 1)['route'] l1.rpc.sendpay(route, inv['payment_hash'], payment_secret=inv['payment_secret']) @@ -3772,3 +3772,18 @@ def test_sql(node_factory, bitcoind): with pytest.raises(RpcError, match='Unauthorized'): l2.rpc.sql("DELETE FROM forwards;") + + assert len(l3.rpc.sql("SELECT * FROM channels;")['rows']) == 4 + # Check that channels gets refreshed! + scid = l1.get_channel_scid(l2) + l1.rpc.setchannel(scid, feebase=123) + wait_for(lambda: l3.rpc.sql("SELECT short_channel_id FROM channels WHERE base_fee_millisatoshi = 123;")['rows'] == [[scid]]) + l3.daemon.wait_for_log("Refreshing channels...") + l3.daemon.wait_for_log("Refreshing channel: {}".format(scid)) + + # This has to wait for the hold_invoice plugin to let go! + l1.rpc.close(l2.info['id']) + bitcoind.generate_block(13, wait_for_mempool=1) + wait_for(lambda: len(l3.rpc.listchannels()['channels']) == 2) + assert len(l3.rpc.sql("SELECT * FROM channels;")['rows']) == 2 + l3.daemon.wait_for_log("Deleting channel: {}".format(scid)) From e531904c36610cd197d6272c526da2fc02d5ea75 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:18 +1030 Subject: [PATCH 28/36] doc/schemas: fix old deprecations. `"deprecated": true` is obsolete; we don't document them anyway. Where it would have otherwise changed the GRPC wrappers, I actually put the version number in. We allow "listchannels" to have "satoshis" since we have some tests that run in deprecated-api mode. Signed-off-by: Rusty Russell --- doc/lightning-decode.7.md | 2 +- doc/lightning-decodepay.7.md | 2 +- doc/lightning-delinvoice.7.md | 2 +- doc/lightning-delpay.7.md | 2 +- doc/lightning-getinfo.7.md | 3 +- doc/lightning-getroute.7.md | 3 +- doc/lightning-keysend.7.md | 2 +- doc/lightning-listchannels.7.md | 2 +- doc/lightning-listforwards.7.md | 2 +- doc/lightning-listfunds.7.md | 2 +- doc/lightning-listinvoices.7.md | 2 +- doc/lightning-listpeerchannels.7.md | 6 +-- doc/lightning-listpeers.7.md | 2 +- doc/lightning-listsendpays.7.md | 2 +- doc/lightning-listtransactions.7.md | 2 +- doc/lightning-pay.7.md | 2 +- doc/lightning-sendinvoice.7.md | 2 +- doc/lightning-sendonion.7.md | 2 +- doc/lightning-sendpay.7.md | 2 +- doc/lightning-waitanyinvoice.7.md | 2 +- doc/lightning-waitinvoice.7.md | 2 +- doc/lightning-waitsendpay.7.md | 2 +- doc/schemas/decode.schema.json | 4 -- doc/schemas/decodepay.schema.json | 4 -- doc/schemas/delinvoice.schema.json | 6 --- doc/schemas/delpay.schema.json | 6 --- doc/schemas/getinfo.schema.json | 2 +- doc/schemas/getroute.schema.json | 2 +- doc/schemas/keysend.schema.json | 6 --- doc/schemas/listchannels.schema.json | 4 +- doc/schemas/listforwards.schema.json | 9 ---- doc/schemas/listfunds.schema.json | 6 --- doc/schemas/listinvoices.schema.json | 6 --- doc/schemas/listpeerchannels.schema.json | 54 ++---------------------- doc/schemas/listpeers.schema.json | 48 --------------------- doc/schemas/listsendpays.schema.json | 6 --- doc/schemas/listtransactions.schema.json | 3 -- doc/schemas/pay.schema.json | 6 --- doc/schemas/sendinvoice.schema.json | 6 --- doc/schemas/sendonion.schema.json | 6 --- doc/schemas/sendpay.request.json | 3 -- doc/schemas/sendpay.schema.json | 6 --- doc/schemas/waitanyinvoice.schema.json | 6 --- doc/schemas/waitinvoice.schema.json | 6 --- doc/schemas/waitsendpay.schema.json | 6 --- 45 files changed, 33 insertions(+), 228 deletions(-) diff --git a/doc/lightning-decode.7.md b/doc/lightning-decode.7.md index 3ae556d4e6fe..5e48491dddea 100644 --- a/doc/lightning-decode.7.md +++ b/doc/lightning-decode.7.md @@ -303,4 +303,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:ba144aad8e34d9a8581161be01fa9a5e0107d068ad35411a278539503446768b) +[comment]: # ( SHA256STAMP:2f77622e54345ebdffbbc0823f73c8f709a29de536be0c84290aac65e5405d3a) diff --git a/doc/lightning-decodepay.7.md b/doc/lightning-decodepay.7.md index f7d6f5d7c77c..119cd63fe396 100644 --- a/doc/lightning-decodepay.7.md +++ b/doc/lightning-decodepay.7.md @@ -71,4 +71,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:d287a96b5495b4be07d8a20633b9a6d5179ef74fc33b1b517c1b201e1b86e9aa) +[comment]: # ( SHA256STAMP:e20f638716d74697afbea9cb4dd5afa380505dda65dcd3bba1579d2ed79bdc6b) diff --git a/doc/lightning-delinvoice.7.md b/doc/lightning-delinvoice.7.md index 69b2b3550efb..345c151f9609 100644 --- a/doc/lightning-delinvoice.7.md +++ b/doc/lightning-delinvoice.7.md @@ -81,4 +81,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:7135589b2dd43b221132efed8753afd2e3ecc0aa9ad5706753fbd6c5b54c1509) +[comment]: # ( SHA256STAMP:f29100ae7e0ad26fee559e175e104a9e29e1a2cd6917c4072d521ce0600d1656) diff --git a/doc/lightning-delpay.7.md b/doc/lightning-delpay.7.md index 810b480121d5..021787d7b8af 100644 --- a/doc/lightning-delpay.7.md +++ b/doc/lightning-delpay.7.md @@ -107,4 +107,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:04fdf8931ea040a3433df9e25b1db1e808e733ad3a5b2586f6edd030ae6f165a) +[comment]: # ( SHA256STAMP:a7736b0f340fce7c02a7bdfeb2c5321656c490a5046129895d6689c2d82cc431) diff --git a/doc/lightning-getinfo.7.md b/doc/lightning-getinfo.7.md index 7d0c237fc665..6cf474652462 100644 --- a/doc/lightning-getinfo.7.md +++ b/doc/lightning-getinfo.7.md @@ -45,6 +45,7 @@ On success, an object is returned, containing: - **node** (hex): features in our node\_announcement message - **channel** (hex): negotiated channel features we (as channel initiator) publish in the channel\_announcement message - **invoice** (hex): features in our BOLT11 invoices +- **msatoshi\_fees\_collected** (u64, optional) **deprecated, removal in v23.05** - **address** (array of objects, optional): The addresses we announce to the world: - **type** (string): Type of connection (one of "dns", "ipv4", "ipv6", "torv2", "torv3", "websocket") - **port** (u16): port number @@ -132,4 +133,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:043b3816857b0dde57f8233b159f2f932dc72dabd532ca5573b6a0e02b9906d1) +[comment]: # ( SHA256STAMP:cd659304258fa31d104fa4bd41855a699a62777e6b57998fdbc064ffe02a293a) diff --git a/doc/lightning-getroute.7.md b/doc/lightning-getroute.7.md index 3c474e045117..9f37ae2c03e0 100644 --- a/doc/lightning-getroute.7.md +++ b/doc/lightning-getroute.7.md @@ -286,6 +286,7 @@ On success, an object containing **route** is returned. It is an array of objec - **amount\_msat** (msat): The amount expected by the node at the end of this hop - **delay** (u32): The total CLTV expected by the node at the end of this hop - **style** (string): The features understood by the destination node (always "tlv") +- **msatoshi** (u64, optional) **deprecated, removal in v23.05** [comment]: # (GENERATE-FROM-SCHEMA-END) @@ -310,4 +311,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:cc32216cf7ace9054b63870c06de74c2870dc336bf194d3081dab9893cd56f58) +[comment]: # ( SHA256STAMP:336fb7d687a26e733ca0cc5f0ca49fb00edfaf311edc43773375201b1180f716) diff --git a/doc/lightning-keysend.7.md b/doc/lightning-keysend.7.md index 60e22428c0b3..6b36d39fee65 100644 --- a/doc/lightning-keysend.7.md +++ b/doc/lightning-keysend.7.md @@ -118,4 +118,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:388b5d185f053b32176eef14bc659c405f56b4096b7635d2eac38583b0285889) +[comment]: # ( SHA256STAMP:326cebe5519961ab09dfc2892aa67932b6c3365394317a630d94b4e10082203e) diff --git a/doc/lightning-listchannels.7.md b/doc/lightning-listchannels.7.md index 3bc691c8139a..4e30be0eb1a8 100644 --- a/doc/lightning-listchannels.7.md +++ b/doc/lightning-listchannels.7.md @@ -80,4 +80,4 @@ Lightning RFC site - BOLT \#7: -[comment]: # ( SHA256STAMP:097b3909247d033ccdc82b5b5bbf222ca391140b4fa160c1d5fae714d06c8dce) +[comment]: # ( SHA256STAMP:cef9786aeca2eddaca0d1adf6dc3d0eef442297e0f63d7c49647e65dbca73396) diff --git a/doc/lightning-listforwards.7.md b/doc/lightning-listforwards.7.md index d129fa3802b9..c03a742a6afd 100644 --- a/doc/lightning-listforwards.7.md +++ b/doc/lightning-listforwards.7.md @@ -64,4 +64,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:10b0eea0c6b65287a913ce0c4c4d73d089e2e45d81e5c360f345bee950db0957) +[comment]: # ( SHA256STAMP:fb6b59740d52aee780678850445bdd58803b33c1df02c5794473ee87c23da35b) diff --git a/doc/lightning-listfunds.7.md b/doc/lightning-listfunds.7.md index 52d181ec44da..197653af9f8e 100644 --- a/doc/lightning-listfunds.7.md +++ b/doc/lightning-listfunds.7.md @@ -73,4 +73,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:d420be5d3e7fa9cf8b21ac928b91ebe08bb68a782e2a04b21b215d81066094f5) +[comment]: # ( SHA256STAMP:3282d4025e161d6926878a5fc8d78384784885749630f007cc5dfcd8d2794b10) diff --git a/doc/lightning-listinvoices.7.md b/doc/lightning-listinvoices.7.md index 4e1de86e457b..9e50acb2c19c 100644 --- a/doc/lightning-listinvoices.7.md +++ b/doc/lightning-listinvoices.7.md @@ -58,4 +58,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:1b31938109207decabf1e0f25cc9607dc03de7f043f4e5fcfbfb8c85ffacec8c) +[comment]: # ( SHA256STAMP:7b1b70f04245395de28eb378e364537920e2f690db3c97ee638cefd282712dca) diff --git a/doc/lightning-listpeerchannels.7.md b/doc/lightning-listpeerchannels.7.md index cc3e59f2b2cd..235183c87c87 100644 --- a/doc/lightning-listpeerchannels.7.md +++ b/doc/lightning-listpeerchannels.7.md @@ -56,8 +56,8 @@ On success, an object containing **channels** is returned. It is an array of ob - **funding** (object, optional): - **local\_funds\_msat** (msat): Amount of channel we funded - **remote\_funds\_msat** (msat): Amount of channel they funded - - **local\_msat** (msat, optional): Amount of channel we funded (deprecated) - - **remote\_msat** (msat, optional): Amount of channel they funded (deprecated) + - **local\_msat** (msat, optional): Amount of channel we funded **deprecated, removal in v23.05** + - **remote\_msat** (msat, optional): Amount of channel they funded **deprecated, removal in v23.05** - **pushed\_msat** (msat, optional): Amount pushed from opener to peer - **fee\_paid\_msat** (msat, optional): Amount we paid peer at open - **fee\_rcvd\_msat** (msat, optional): Amount we were paid by peer at open @@ -191,4 +191,4 @@ Main web site: Lightning RFC site (BOLT \#9): -[comment]: # ( SHA256STAMP:4abe4d7c2e43629d536f4ef906c579a36e2636b183692ffee1474a18438ab630) +[comment]: # ( SHA256STAMP:32eef1dd02f6bdd40e8d81057701e8170fac788f4396e34f5f505efbed360245) diff --git a/doc/lightning-listpeers.7.md b/doc/lightning-listpeers.7.md index a3301345057f..0d4ac5876491 100644 --- a/doc/lightning-listpeers.7.md +++ b/doc/lightning-listpeers.7.md @@ -399,4 +399,4 @@ Main web site: Lightning RFC site (BOLT \#9): -[comment]: # ( SHA256STAMP:696c0d20a0fdc2a40531be5f38a9460272c4ff70667433fd08da3309b74286d5) +[comment]: # ( SHA256STAMP:b89450ac6f27e051003bcf0a382b51e117e2832729e9d80b0015d9cebfacfa2c) diff --git a/doc/lightning-listsendpays.7.md b/doc/lightning-listsendpays.7.md index 53feb23927f0..63d9f2207008 100644 --- a/doc/lightning-listsendpays.7.md +++ b/doc/lightning-listsendpays.7.md @@ -65,4 +65,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:9de82a239bae09bf777bdb988170c7ec43946ea49c9dfa908430f65d0a42fdbb) +[comment]: # ( SHA256STAMP:635a4026d5472207c545391db99c4f5569ad5388ada009de028a0b4063c594a4) diff --git a/doc/lightning-listtransactions.7.md b/doc/lightning-listtransactions.7.md index ea5a71228c56..05fce8ea5dd2 100644 --- a/doc/lightning-listtransactions.7.md +++ b/doc/lightning-listtransactions.7.md @@ -103,4 +103,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:b3b7286a81cdae413baac015e87d65a095506accb32ecea5dc1fdccdc8e73c4c) +[comment]: # ( SHA256STAMP:525f24511eb9687dc16d5b2156d4d8df28b371e287512a749d2d9dfd5701e093) diff --git a/doc/lightning-pay.7.md b/doc/lightning-pay.7.md index 4ef5d9788065..c0a02dff99f4 100644 --- a/doc/lightning-pay.7.md +++ b/doc/lightning-pay.7.md @@ -168,4 +168,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:d220052ef3c013560eb3b4b379a5f5aa5ff4ce719b0bd2f05f0645cfc25804f9) +[comment]: # ( SHA256STAMP:f72845c2600efdf48d5c9d32be5f3154c48bd5852df28b3a941f8e7f65bd1193) diff --git a/doc/lightning-sendinvoice.7.md b/doc/lightning-sendinvoice.7.md index d4a9c424396a..0258b92ae835 100644 --- a/doc/lightning-sendinvoice.7.md +++ b/doc/lightning-sendinvoice.7.md @@ -80,4 +80,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:7646a92936e1e79dbefe9cacf418ee15f148db467d780a9b39b90d46ca522539) +[comment]: # ( SHA256STAMP:79a371e3dc60c1395f591e16c3dd039f8fdee53c9402f345fa8823d85a18d819) diff --git a/doc/lightning-sendonion.7.md b/doc/lightning-sendonion.7.md index 9242a01de389..4d9a7079a27f 100644 --- a/doc/lightning-sendonion.7.md +++ b/doc/lightning-sendonion.7.md @@ -135,4 +135,4 @@ RESOURCES Main web site: [bolt04]: https://github.com/lightning/bolts/blob/master/04-onion-routing.md -[comment]: # ( SHA256STAMP:ffc19dc92199d296f1f8bc974923abd76378f361ff68697137b9dab864d65094) +[comment]: # ( SHA256STAMP:c1f3def8b395cd7d56a8a9270c46027d8b097a124a010931006926f6322257c5) diff --git a/doc/lightning-sendpay.7.md b/doc/lightning-sendpay.7.md index 7051e0b563b3..855c4968dd84 100644 --- a/doc/lightning-sendpay.7.md +++ b/doc/lightning-sendpay.7.md @@ -142,4 +142,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:1cc526944ea1119f507383f58a9c251dff2ca0b86c15675317753328549be78d) +[comment]: # ( SHA256STAMP:b4bbebdb6b9de7aa6fa2ba6949cd9e38576dbd9665cd0d1eabc64e0782590f53) diff --git a/doc/lightning-waitanyinvoice.7.md b/doc/lightning-waitanyinvoice.7.md index 850a0111c730..4f9f05dcc9df 100644 --- a/doc/lightning-waitanyinvoice.7.md +++ b/doc/lightning-waitanyinvoice.7.md @@ -75,4 +75,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:455c57c49af09b9360b8c4a052828ce71f75cb178a63646eb2621e9e9f0faa5a) +[comment]: # ( SHA256STAMP:86e3d74f12d2997b7960a0d0732ff51422af6dc9851134e216723b1497bc0573) diff --git a/doc/lightning-waitinvoice.7.md b/doc/lightning-waitinvoice.7.md index f2a8767949bb..0945d6538e33 100644 --- a/doc/lightning-waitinvoice.7.md +++ b/doc/lightning-waitinvoice.7.md @@ -60,4 +60,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:455c57c49af09b9360b8c4a052828ce71f75cb178a63646eb2621e9e9f0faa5a) +[comment]: # ( SHA256STAMP:86e3d74f12d2997b7960a0d0732ff51422af6dc9851134e216723b1497bc0573) diff --git a/doc/lightning-waitsendpay.7.md b/doc/lightning-waitsendpay.7.md index 61f6de122be4..f6e412ce90fa 100644 --- a/doc/lightning-waitsendpay.7.md +++ b/doc/lightning-waitsendpay.7.md @@ -104,4 +104,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:42da651674955a4e839abc7677263fbdda3a62ba3ce7c251b698828d604b0d4c) +[comment]: # ( SHA256STAMP:d9ed7646579daf789b74686b18a09145ece3f1a0ebfc5dc579f91652ae187276) diff --git a/doc/schemas/decode.schema.json b/doc/schemas/decode.schema.json index 39c1b361cd10..92b0ac047643 100644 --- a/doc/schemas/decode.schema.json +++ b/doc/schemas/decode.schema.json @@ -1259,10 +1259,6 @@ "type": "pubkey", "description": "the public key of the recipient" }, - "msatoshi": { - "type": "u64", - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "Amount the invoice asked for" diff --git a/doc/schemas/decodepay.schema.json b/doc/schemas/decodepay.schema.json index e1d536e2848d..bb2547d6c4e8 100644 --- a/doc/schemas/decodepay.schema.json +++ b/doc/schemas/decodepay.schema.json @@ -28,10 +28,6 @@ "type": "pubkey", "description": "the public key of the recipient" }, - "msatoshi": { - "type": "u64", - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "Amount the invoice asked for" diff --git a/doc/schemas/delinvoice.schema.json b/doc/schemas/delinvoice.schema.json index 92d7eac267f5..a003e92661ed 100644 --- a/doc/schemas/delinvoice.schema.json +++ b/doc/schemas/delinvoice.schema.json @@ -21,9 +21,6 @@ "type": "string", "description": "BOLT12 string" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "the amount required to pay this invoice" @@ -144,9 +141,6 @@ "type": "msat", "description": "how much was actually received" }, - "msatoshi_received": { - "deprecated": true - }, "paid_at": { "type": "u64", "description": "UNIX timestamp of when payment was received" diff --git a/doc/schemas/delpay.schema.json b/doc/schemas/delpay.schema.json index d1d6f81e77ce..43c0fcd95105 100644 --- a/doc/schemas/delpay.schema.json +++ b/doc/schemas/delpay.schema.json @@ -36,9 +36,6 @@ ], "description": "status of the payment" }, - "msatoshi_sent": { - "deprecated": true - }, "amount_sent_msat": { "type": "msat", "description": "the amount we actually sent, including fees" @@ -51,9 +48,6 @@ "type": "pubkey", "description": "the final destination of the payment if known" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "the amount the destination received, if known" diff --git a/doc/schemas/getinfo.schema.json b/doc/schemas/getinfo.schema.json index 21001d0c78dd..665306638422 100644 --- a/doc/schemas/getinfo.schema.json +++ b/doc/schemas/getinfo.schema.json @@ -95,7 +95,7 @@ }, "msatoshi_fees_collected": { "type": "u64", - "deprecated": true + "deprecated": "v0.12.0" }, "fees_collected_msat": { "type": "msat", diff --git a/doc/schemas/getroute.schema.json b/doc/schemas/getroute.schema.json index 586e4d556975..6e061415e26e 100644 --- a/doc/schemas/getroute.schema.json +++ b/doc/schemas/getroute.schema.json @@ -34,7 +34,7 @@ }, "msatoshi": { "type": "u64", - "deprecated": true + "deprecated": "v0.12.0" }, "amount_msat": { "type": "msat", diff --git a/doc/schemas/keysend.schema.json b/doc/schemas/keysend.schema.json index 560d5dc58652..a2a3fc88b435 100644 --- a/doc/schemas/keysend.schema.json +++ b/doc/schemas/keysend.schema.json @@ -32,16 +32,10 @@ "type": "u32", "description": "how many attempts this took" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "Amount the recipient received" }, - "msatoshi_sent": { - "deprecated": true - }, "amount_sent_msat": { "type": "msat", "description": "Total amount we sent (including fees)" diff --git a/doc/schemas/listchannels.schema.json b/doc/schemas/listchannels.schema.json index acc08380647b..9761f0a15e8f 100644 --- a/doc/schemas/listchannels.schema.json +++ b/doc/schemas/listchannels.schema.json @@ -85,9 +85,7 @@ "type": "msat", "description": "The smallest payment *source* will allow via this channel" }, - "satoshis": { - "deprecated": true - }, + "satoshis": {}, "htlc_maximum_msat": { "type": "msat", "description": "The largest payment *source* will allow via this channel" diff --git a/doc/schemas/listforwards.schema.json b/doc/schemas/listforwards.schema.json index 9726a5dc34f1..c18f8816be08 100644 --- a/doc/schemas/listforwards.schema.json +++ b/doc/schemas/listforwards.schema.json @@ -26,9 +26,6 @@ "type": "u64", "description": "the unique HTLC id the sender gave this (not present if incoming channel was closed before ugprade to v22.11)" }, - "in_msatoshi": { - "deprecated": true - }, "in_msat": { "type": "msat", "description": "the value of the incoming HTLC" @@ -91,16 +88,10 @@ "out_htlc_id": {}, "failcode": {}, "failreason": {}, - "fee": { - "deprecated": true - }, "fee_msat": { "type": "msat", "description": "the amount this paid in fees" }, - "out_msatoshi": { - "deprecated": true - }, "out_msat": { "type": "msat", "description": "the amount we sent out the *out_channel*" diff --git a/doc/schemas/listfunds.schema.json b/doc/schemas/listfunds.schema.json index 4c02c72e2e1c..613d005d24f6 100644 --- a/doc/schemas/listfunds.schema.json +++ b/doc/schemas/listfunds.schema.json @@ -154,16 +154,10 @@ "type": "msat", "description": "available satoshis on our node's end of the channel" }, - "channel_sat": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "total channel value" }, - "channel_total_sat": { - "deprecated": true - }, "funding_txid": { "type": "txid", "description": "funding transaction id" diff --git a/doc/schemas/listinvoices.schema.json b/doc/schemas/listinvoices.schema.json index c18b48ef0b6e..ccc61eca264d 100644 --- a/doc/schemas/listinvoices.schema.json +++ b/doc/schemas/listinvoices.schema.json @@ -43,9 +43,6 @@ "type": "u64", "description": "UNIX timestamp of when it will become / became unpayable" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "the amount required to pay this invoice" @@ -103,9 +100,6 @@ "type": "u64", "description": "Unique incrementing index for this payment" }, - "msatoshi_received": { - "deprecated": true - }, "amount_received_msat": { "type": "msat", "description": "the amount actually received (could be slightly greater than *amount_msat*, since clients may overpay)" diff --git a/doc/schemas/listpeerchannels.schema.json b/doc/schemas/listpeerchannels.schema.json index 7764716bf5dc..d3fb4c108184 100644 --- a/doc/schemas/listpeerchannels.schema.json +++ b/doc/schemas/listpeerchannels.schema.json @@ -191,11 +191,13 @@ "properties": { "local_msat": { "type": "msat", - "description": "Amount of channel we funded (deprecated)" + "deprecated": "v0.12.0", + "description": "Amount of channel we funded" }, "remote_msat": { "type": "msat", - "description": "Amount of channel they funded (deprecated)" + "deprecated": "v0.12.0", + "description": "Amount of channel they funded" }, "pushed_msat": { "type": "msat", @@ -291,39 +293,6 @@ "type": "u32", "description": "Maximum number of incoming HTLC we will accept at once" }, - "msatoshi_to_us": { - "deprecated": true - }, - "msatoshi_to_us_min": { - "deprecated": true - }, - "msatoshi_to_us_max": { - "deprecated": true - }, - "msatoshi_total": { - "deprecated": true - }, - "dust_limit_satoshis": { - "deprecated": true - }, - "max_htlc_value_in_flight_msat": { - "deprecated": true - }, - "our_channel_reserve_satoshis": { - "deprecated": true - }, - "their_channel_reserve_satoshis": { - "deprecated": true - }, - "spendable_msatoshi": { - "deprecated": true - }, - "receivable_msatoshi": { - "deprecated": true - }, - "htlc_minimum_msat": { - "deprecated": true - }, "alias": { "type": "object", "required": [], @@ -424,9 +393,6 @@ "type": "msat", "description": "Total amount of incoming payment attempts" }, - "in_msatoshi_offered": { - "deprecated": true - }, "in_payments_fulfilled": { "type": "u64", "description": "Number of successful incoming payment attempts" @@ -435,9 +401,6 @@ "type": "msat", "description": "Total amount of successful incoming payment attempts" }, - "in_msatoshi_fulfilled": { - "deprecated": true - }, "out_payments_offered": { "type": "u64", "description": "Number of outgoing payment attempts" @@ -446,9 +409,6 @@ "type": "msat", "description": "Total amount of outgoing payment attempts" }, - "out_msatoshi_offered": { - "deprecated": true - }, "out_payments_fulfilled": { "type": "u64", "description": "Number of successful outgoing payment attempts" @@ -457,9 +417,6 @@ "type": "msat", "description": "Total amount of successful outgoing payment attempts" }, - "out_msatoshi_fulfilled": { - "deprecated": true - }, "htlcs": { "type": "array", "description": "current HTLCs in this channel", @@ -491,9 +448,6 @@ "type": "msat", "description": "Amount send/received for this HTLC" }, - "msatoshi": { - "deprecated": true - }, "expiry": { "type": "u32", "description": "Block this HTLC expires at (after which an `in` direction HTLC will be returned to the peer, an `out` returned to us). If this expiry is too close, lightningd(8) will automatically unilaterally close the channel in order to enforce the timeout onchain." diff --git a/doc/schemas/listpeers.schema.json b/doc/schemas/listpeers.schema.json index 4087cfc02a8a..6358994ed3d9 100644 --- a/doc/schemas/listpeers.schema.json +++ b/doc/schemas/listpeers.schema.json @@ -445,39 +445,6 @@ "type": "u32", "description": "Maximum number of incoming HTLC we will accept at once" }, - "msatoshi_to_us": { - "deprecated": true - }, - "msatoshi_to_us_min": { - "deprecated": true - }, - "msatoshi_to_us_max": { - "deprecated": true - }, - "msatoshi_total": { - "deprecated": true - }, - "dust_limit_satoshis": { - "deprecated": true - }, - "max_htlc_value_in_flight_msat": { - "deprecated": true - }, - "our_channel_reserve_satoshis": { - "deprecated": true - }, - "their_channel_reserve_satoshis": { - "deprecated": true - }, - "spendable_msatoshi": { - "deprecated": true - }, - "receivable_msatoshi": { - "deprecated": true - }, - "htlc_minimum_msat": { - "deprecated": true - }, "alias": { "type": "object", "required": [], @@ -578,9 +545,6 @@ "type": "msat", "description": "Total amount of incoming payment attempts" }, - "in_msatoshi_offered": { - "deprecated": true - }, "in_payments_fulfilled": { "type": "u64", "description": "Number of successful incoming payment attempts" @@ -589,9 +553,6 @@ "type": "msat", "description": "Total amount of successful incoming payment attempts" }, - "in_msatoshi_fulfilled": { - "deprecated": true - }, "out_payments_offered": { "type": "u64", "description": "Number of outgoing payment attempts" @@ -600,9 +561,6 @@ "type": "msat", "description": "Total amount of outgoing payment attempts" }, - "out_msatoshi_offered": { - "deprecated": true - }, "out_payments_fulfilled": { "type": "u64", "description": "Number of successful outgoing payment attempts" @@ -611,9 +569,6 @@ "type": "msat", "description": "Total amount of successful outgoing payment attempts" }, - "out_msatoshi_fulfilled": { - "deprecated": true - }, "htlcs": { "type": "array", "description": "current HTLCs in this channel", @@ -645,9 +600,6 @@ "type": "msat", "description": "Amount send/received for this HTLC" }, - "msatoshi": { - "deprecated": true - }, "expiry": { "type": "u32", "description": "Block this HTLC expires at" diff --git a/doc/schemas/listsendpays.schema.json b/doc/schemas/listsendpays.schema.json index 65f56daf3cf1..8a9ad68abe02 100644 --- a/doc/schemas/listsendpays.schema.json +++ b/doc/schemas/listsendpays.schema.json @@ -45,9 +45,6 @@ ], "description": "status of the payment" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "The amount delivered to destination (if known)" @@ -60,9 +57,6 @@ "type": "u64", "description": "the UNIX timestamp showing when this payment was initiated" }, - "msatoshi_sent": { - "deprecated": true - }, "amount_sent_msat": { "type": "msat", "description": "The amount sent" diff --git a/doc/schemas/listtransactions.schema.json b/doc/schemas/listtransactions.schema.json index e0f75bd66368..0edcb7ba1f2a 100644 --- a/doc/schemas/listtransactions.schema.json +++ b/doc/schemas/listtransactions.schema.json @@ -114,9 +114,6 @@ "type": "msat", "description": "the amount of the output" }, - "msat": { - "deprecated": true - }, "scriptPubKey": { "type": "hex", "description": "the scriptPubKey" diff --git a/doc/schemas/pay.schema.json b/doc/schemas/pay.schema.json index 13d7d7612e22..697622643032 100644 --- a/doc/schemas/pay.schema.json +++ b/doc/schemas/pay.schema.json @@ -32,16 +32,10 @@ "type": "u32", "description": "how many attempts this took" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "Amount the recipient received" }, - "msatoshi_sent": { - "deprecated": true - }, "amount_sent_msat": { "type": "msat", "description": "Total amount we sent (including fees)" diff --git a/doc/schemas/sendinvoice.schema.json b/doc/schemas/sendinvoice.schema.json index bba697e445cf..709eaa78a138 100644 --- a/doc/schemas/sendinvoice.schema.json +++ b/doc/schemas/sendinvoice.schema.json @@ -35,9 +35,6 @@ "type": "u64", "description": "UNIX timestamp of when it will become / became unpayable" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "the amount required to pay this invoice" @@ -80,9 +77,6 @@ "type": "u64", "description": "Unique incrementing index for this payment" }, - "msatoshi_received": { - "deprecated": true - }, "amount_received_msat": { "type": "msat", "description": "the amount actually received (could be slightly greater than *amount_msat*, since clients may overpay)" diff --git a/doc/schemas/sendonion.schema.json b/doc/schemas/sendonion.schema.json index 8a49a5c19ad2..46815a9b8c3f 100644 --- a/doc/schemas/sendonion.schema.json +++ b/doc/schemas/sendonion.schema.json @@ -26,9 +26,6 @@ ], "description": "status of the payment (could be complete if already sent previously)" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "The amount delivered to destination (if known)" @@ -41,9 +38,6 @@ "type": "u64", "description": "the UNIX timestamp showing when this payment was initiated" }, - "msatoshi_sent": { - "deprecated": true - }, "amount_sent_msat": { "type": "msat", "description": "The amount sent" diff --git a/doc/schemas/sendpay.request.json b/doc/schemas/sendpay.request.json index 468bdf48aa45..67d645423ec9 100644 --- a/doc/schemas/sendpay.request.json +++ b/doc/schemas/sendpay.request.json @@ -21,9 +21,6 @@ "amount_msat": { "type": "msat" }, - "msatoshi": { - "deprecated": true - }, "id": { "type": "pubkey" }, diff --git a/doc/schemas/sendpay.schema.json b/doc/schemas/sendpay.schema.json index 5cda57faea66..79b9263c4d18 100644 --- a/doc/schemas/sendpay.schema.json +++ b/doc/schemas/sendpay.schema.json @@ -30,9 +30,6 @@ ], "description": "status of the payment (could be complete if already sent previously)" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "The amount delivered to destination (if known)" @@ -49,9 +46,6 @@ "type": "u64", "description": "the UNIX timestamp showing when this payment was completed" }, - "msatoshi_sent": { - "deprecated": true - }, "amount_sent_msat": { "type": "msat", "description": "The amount sent" diff --git a/doc/schemas/waitanyinvoice.schema.json b/doc/schemas/waitanyinvoice.schema.json index 1a1802a18074..583d0f16a71d 100644 --- a/doc/schemas/waitanyinvoice.schema.json +++ b/doc/schemas/waitanyinvoice.schema.json @@ -34,9 +34,6 @@ "type": "u64", "description": "UNIX timestamp of when it will become / became unpayable" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "the amount required to pay this invoice" @@ -84,9 +81,6 @@ "type": "u64", "description": "Unique incrementing index for this payment" }, - "msatoshi_received": { - "deprecated": true - }, "amount_received_msat": { "type": "msat", "description": "the amount actually received (could be slightly greater than *amount_msat*, since clients may overpay)" diff --git a/doc/schemas/waitinvoice.schema.json b/doc/schemas/waitinvoice.schema.json index 1a1802a18074..583d0f16a71d 100644 --- a/doc/schemas/waitinvoice.schema.json +++ b/doc/schemas/waitinvoice.schema.json @@ -34,9 +34,6 @@ "type": "u64", "description": "UNIX timestamp of when it will become / became unpayable" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "the amount required to pay this invoice" @@ -84,9 +81,6 @@ "type": "u64", "description": "Unique incrementing index for this payment" }, - "msatoshi_received": { - "deprecated": true - }, "amount_received_msat": { "type": "msat", "description": "the amount actually received (could be slightly greater than *amount_msat*, since clients may overpay)" diff --git a/doc/schemas/waitsendpay.schema.json b/doc/schemas/waitsendpay.schema.json index 3b73bec93388..5b4607d9e285 100644 --- a/doc/schemas/waitsendpay.schema.json +++ b/doc/schemas/waitsendpay.schema.json @@ -29,9 +29,6 @@ ], "description": "status of the payment" }, - "msatoshi": { - "deprecated": true - }, "amount_msat": { "type": "msat", "description": "The amount delivered to destination (if known)" @@ -48,9 +45,6 @@ "type": "number", "description": "the UNIX timestamp showing when this payment was completed" }, - "msatoshi_sent": { - "deprecated": true - }, "amount_sent_msat": { "type": "msat", "description": "The amount sent" From 4885ef66149088e2b45c7780173f4eac69b699e1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:18 +1030 Subject: [PATCH 29/36] plugins/sql: pay attention to `deprecated` in schema. For now, we ignore every deprecated field, but put in the logic so that future deprecations will work as expected. Signed-off-by: Rusty Russell --- plugins/sql.c | 30 +++++++++++++++++++++++++++++- tests/test_plugin.py | 27 +++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index 00d5a98fea09..7d604dcae8ae 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1138,6 +1138,27 @@ static void add_table_singleton(struct table_desc *td, tal_arr_expand(&td->columns, col); } +static bool is_deprecated(const jsmntok_t *deprecated_tok) +{ + const char *deprstr; + + if (!deprecated_tok) + return false; + + /* If deprecated APIs are globally disabled, we don't want them! */ + if (!deprecated_apis) + return true; + + /* If it was deprecated before our release, we don't want it; older ones + * were simply 'deprecated: true' */ + deprstr = json_strdup(tmpctx, schemas, deprecated_tok); + assert(strstarts(deprstr, "v")); + if (streq(deprstr, "v0.12.0") || streq(deprstr, "v23.02")) + return true; + + return false; +} + static void add_table_properties(struct table_desc *td, const jsmntok_t *properties) { @@ -1145,7 +1166,7 @@ static void add_table_properties(struct table_desc *td, size_t i; json_for_each_obj(i, t, properties) { - const jsmntok_t *type; + const jsmntok_t *type, *deprecated_tok; struct column col; if (ignore_column(td, t)) @@ -1155,6 +1176,13 @@ static void add_table_properties(struct table_desc *td, * another branch with actual types, so ignore this */ if (!type) continue; + + /* Depends on when it was deprecated, and whether deprecations + * are enabled! */ + deprecated_tok = json_get_member(schemas, t+1, "deprecated"); + if (is_deprecated(deprecated_tok)) + continue; + if (json_tok_streq(schemas, type, "array")) { const jsmntok_t *items; diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 484db96a0f1b..237771fbce85 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3539,10 +3539,6 @@ def test_sql(node_factory, bitcoind): 'type': 'string'}, {'name': 'closer', 'type': 'string'}, - {'name': 'funding_local_msat', - 'type': 'msat'}, - {'name': 'funding_remote_msat', - 'type': 'msat'}, {'name': 'funding_pushed_msat', 'type': 'msat'}, {'name': 'funding_local_funds_msat', @@ -3787,3 +3783,26 @@ def test_sql(node_factory, bitcoind): wait_for(lambda: len(l3.rpc.listchannels()['channels']) == 2) assert len(l3.rpc.sql("SELECT * FROM channels;")['rows']) == 2 l3.daemon.wait_for_log("Deleting channel: {}".format(scid)) + + # No deprecated fields! + with pytest.raises(RpcError, match='query failed with no such column: funding_local_msat'): + l2.rpc.sql("SELECT funding_local_msat FROM peerchannels;") + + with pytest.raises(RpcError, match='query failed with no such column: funding_remote_msat'): + l2.rpc.sql("SELECT funding_remote_msat FROM peerchannels;") + + with pytest.raises(RpcError, match='query failed with no such table: peers_channels'): + l2.rpc.sql("SELECT * FROM peers_channels;") + + +def test_sql_deprecated(node_factory, bitcoind): + # deprecated-apis breaks schemas... + l1 = node_factory.get_node(start=False, options={'allow-deprecated-apis': True}) + l1.rpc.check_request_schemas = False + l1.start() + + # FIXME: we have no fields which have been deprecated since sql plugin was + # introduced. When we do, add them here! (I manually tested a fake one) + + # ret = l1.rpc.sql("SELECT funding_local_msat, funding_remote_msat FROM peerchannels;") + # assert ret == {'rows': []} From 472d1b2c05e2ee1705075a5e25cdf43b19750d3f Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:18 +1030 Subject: [PATCH 30/36] plugins/sql: print out part of man page referring to schemas. We now add tables to the strmap as we allocate them, since we don't want to call "finish_td" when we're merely invoked for the documentation, and don't need a database. Signed-off-by: Rusty Russell --- plugins/sql.c | 120 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 9 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index 7d604dcae8ae..b03266bfd8f9 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -8,12 +8,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -25,8 +27,6 @@ static const char schemas[] = /* TODO: * 2. Refresh time in API. - * 3. Colnames API to return dict. - * 4. sql-schemas command. * 5. documentation. * 6. test on mainnet. * 7. Some cool query for documentation. @@ -94,9 +94,10 @@ struct db_query { }; struct table_desc { - /* e.g. listpeers */ + /* e.g. listpeers. For sub-tables, the raw name without + * parent prepended */ const char *cmdname; - /* e.g. peers for listpeers */ + /* e.g. peers for listpeers, peers_channels for listpeers.channels. */ const char *name; /* e.g. "payments" for listsendpays */ const char *arrname; @@ -969,7 +970,7 @@ static bool ignore_column(const struct table_desc *td, const jsmntok_t *t) return false; } -/* Creates sql statements, initializes table, adds to tablemap */ +/* Creates sql statements, initializes table */ static void finish_td(struct plugin *plugin, struct table_desc *td) { char *create_stmt; @@ -1027,8 +1028,6 @@ static void finish_td(struct plugin *plugin, struct table_desc *td) if (err != SQLITE_OK) plugin_err(plugin, "Could not create %s: %s", td->name, errmsg); - strmap_add(&tablemap, td->name, td); - /* Now do any children */ for (size_t i = 0; i < tal_count(td->columns); i++) { const struct column *col = &td->columns[i]; @@ -1101,6 +1100,11 @@ static struct table_desc *new_table_desc(struct table_desc *parent, td->refresh = nodes_refresh; else td->refresh = default_refresh; + + /* sub-objects are a JSON thing, not a real table! */ + if (!td->is_subobject) + strmap_add(&tablemap, td->name, td); + return td; } @@ -1238,6 +1242,7 @@ static void add_table_object(struct table_desc *td, const jsmntok_t *tok) add_table_object(td, cond); } +/* plugin is NULL if we're just doing --print-docs */ static void init_tablemap(struct plugin *plugin) { const jsmntok_t *toks, *t; @@ -1259,10 +1264,14 @@ static void init_tablemap(struct plugin *plugin) assert(json_tok_streq(schemas, type, "object")); td = new_table_desc(NULL, t, cmd, false); - tal_steal(plugin, td); + if (plugin) + tal_steal(plugin, td); + else + tal_steal(tmpctx, td); add_table_object(td, items); - finish_td(plugin, td); + if (plugin) + finish_td(plugin, td); } } @@ -1315,9 +1324,102 @@ static const struct plugin_command commands[] = { { }, }; +static const char *fmt_indexes(const tal_t *ctx, const char *table) +{ + char *ret = NULL; + + for (size_t i = 0; i < ARRAY_SIZE(indices); i++) { + if (!streq(indices[i].tablename, table)) + continue; + /* FIXME: Handle multiple indices! */ + assert(!ret); + BUILD_ASSERT(ARRAY_SIZE(indices[i].fields) == 2); + if (indices[i].fields[1]) + ret = tal_fmt(tmpctx, "%s and %s", + indices[i].fields[0], + indices[i].fields[1]); + else + ret = tal_fmt(tmpctx, "%s", + indices[i].fields[0]); + } + if (!ret) + return ""; + return tal_fmt(ctx, " indexed by `%s`", ret); +} + +static void print_columns(const struct table_desc *td, const char *indent, + const char *objsrc) +{ + for (size_t i = 0; i < tal_count(td->columns); i++) { + const char *origin; + if (td->columns[i].sub) { + const struct table_desc *subtd = td->columns[i].sub; + + if (!subtd->is_subobject) { + const char *subindent; + + subindent = tal_fmt(tmpctx, "%s ", indent); + printf("%s- related table `%s`%s\n", + indent, subtd->name, objsrc); + printf("%s- `row` (reference to `%s.rowid`, sqltype `INTEGER`)\n" + "%s- `arrindex` (index within array, sqltype `INTEGER`)\n", + subindent, td->name, subindent); + print_columns(subtd, subindent, ""); + } else { + const char *subobjsrc; + + subobjsrc = tal_fmt(tmpctx, + ", from JSON object `%s`", + td->columns[i].jsonname); + print_columns(subtd, indent, subobjsrc); + } + continue; + } + + if (streq(objsrc, "") + && td->columns[i].jsonname + && !streq(td->columns[i].dbname, td->columns[i].jsonname)) { + origin = tal_fmt(tmpctx, ", from JSON field `%s`", + td->columns[i].jsonname); + } else + origin = ""; + printf("%s- `%s` (type `%s`, sqltype `%s`%s%s)\n", + indent, td->columns[i].dbname, + fieldtypemap[td->columns[i].ftype].name, + fieldtypemap[td->columns[i].ftype].sqltype, + origin, objsrc); + } +} + +static bool print_one_table(const char *member, + struct table_desc *td, + void *unused) +{ + if (td->parent) + return true; + + printf("- `%s`%s (see lightning-%s(7))\n", + member, fmt_indexes(tmpctx, member), td->cmdname); + + print_columns(td, " ", ""); + printf("\n"); + return true; +} + int main(int argc, char *argv[]) { setup_locale(); + + if (argc == 2 && streq(argv[1], "--print-docs")) { + common_setup(argv[0]); + /* plugin is NULL, so just sets up tables */ + init_tablemap(NULL); + + printf("The following tables are currently supported:\n"); + strmap_iterate(&tablemap, print_one_table, NULL); + common_shutdown(); + return 0; + } plugin_main(argv, init, PLUGIN_RESTARTABLE, true, NULL, commands, ARRAY_SIZE(commands), NULL, 0, NULL, 0, NULL, 0, plugin_option("sqlfilename", From 224383b7577fe154d420a263132e524eaac71fa1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:18 +1030 Subject: [PATCH 31/36] doc: document the sql command. In particular, we generate the schema part from the plugin itself. Signed-off-by: Rusty Russell Changelog-Added: Plugins: `sql` plugin command to perform server-side complex queries. --- doc/.gitignore | 1 + doc/Makefile | 9 + doc/index.rst | 1 + doc/lightning-sql.7.md | 315 +++++++++++++++++++++++++++++++++++ doc/schemas/sql.request.json | 12 ++ doc/schemas/sql.schema.json | 20 +++ plugins/sql.c | 1 - 7 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 doc/lightning-sql.7.md create mode 100644 doc/schemas/sql.request.json create mode 100644 doc/schemas/sql.schema.json diff --git a/doc/.gitignore b/doc/.gitignore index 4a4d6a7565d9..d1b31fbe759c 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -5,3 +5,4 @@ *.log *.out *.tex +.sqlgen diff --git a/doc/Makefile b/doc/Makefile index 780ff6b3b434..174a94131f0d 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -85,6 +85,7 @@ MANPAGES := doc/lightning-cli.1 \ doc/lightning-setchannel.7 \ doc/lightning-sendcustommsg.7 \ doc/lightning-signmessage.7 \ + doc/lightning-sql.7 \ doc/lightning-staticbackup.7 \ doc/lightning-txprepare.7 \ doc/lightning-txdiscard.7 \ @@ -149,6 +150,14 @@ $(MANPAGES): doc/%: doc/%.md tools/md2man.sh version_gen.h $(MANPAGES): $(FORCE) $(MARKDOWN_WITH_SCHEMA): $(FORCE) +# Use awk for preamble, then again for post, with the new docs in the middle. +# We can't put plugins/sql in deps directly, since they all get sha256! +doc/.sqlgen: plugins/sql + @plugins/sql --print-docs > $@ + +doc/lightning-sql.7.md: doc/.sqlgen $(FORCE) + @if $(call SHA256STAMP_CHANGED); then $(call VERBOSE, "sql-print-docs $@", awk "/GENERATE-DOC-START/ { print \$$0; exit } { print \$$0 }" < $@ > $@.tmp && cat doc/.sqlgen >> $@.tmp && (awk "/GENERATE-DOC-END/ { PRINT=1 } { if (PRINT) { print \$$0 } }" | grep -v SHA256STAMP) < $@ >> $@.tmp && mv $@.tmp $@ && $(call SHA256STAMP,[comment]: # $(LBRACKET),$(RBRACKET))); else touch $@; fi + doc/protocol-%.svg: test/test_protocol test/test_protocol --svg < test/commits/$*.script > $@ diff --git a/doc/index.rst b/doc/index.rst index 3a7907e198c6..822679c109aa 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -117,6 +117,7 @@ Core Lightning Documentation lightning-setchannel lightning-signmessage lightning-signpsbt + lightning-sql lightning-staticbackup lightning-stop lightning-txdiscard diff --git a/doc/lightning-sql.7.md b/doc/lightning-sql.7.md new file mode 100644 index 000000000000..bb645244acc9 --- /dev/null +++ b/doc/lightning-sql.7.md @@ -0,0 +1,315 @@ +lightning-sql -- Command to do complex queries on list commands +=============================================================== + +SYNOPSIS +-------- + +**sql** *query* + +DESCRIPTION +----------- + +The **sql** RPC command runs the given query across a sqlite3 database +created from various list commands. + +When tables are accessed, it calls the above commands, so it's no +faster than any other local access (though it goes to great length to +cache `listnodes` and `listchannels`) which then processes the results. + +It is, however faster for remote access if the result of the query is +much smaller than the list commands would be. + +TREATMENT OF TYPES +------------------ + +The following types are supported in schemas, and this shows how they +are presented in the database. This matters: a JSON boolean is +represented as an integer in the database, so a query will return 0 or +1, not true or false. + +* *hex*. A hex string. + * JSON: a string + * sqlite3: BLOB + +* *hash*/*secret*/*pubkey*/*txid*: just like *hex*. + +* *msat*/*integer*/*u64*/*u32*/*u16*/*u8*. Normal numbers. + * JSON: an unsigned integer + * sqlite3: INTEGER + +* *boolean*. True or false. + * JSON: literal **true** or **false** + * sqlite3: INTEGER + +* *number*. A floating point number (used for times in some places). + * JSON: number + * sqlite3: REAL + +* *string*. Text. + * JSON: string + * sqlite3: TEXT + +* *short\_channel\_id*. A short-channel-id of form 1x2x3. + * JSON: string + * sqlite3: TEXT + +TABLES +------ +[comment]: # (GENERATE-DOC-START) +The following tables are currently supported: +- `channels` indexed by `short_channel_id` (see lightning-listchannels(7)) + - `source` (type `pubkey`, sqltype `BLOB`) + - `destination` (type `pubkey`, sqltype `BLOB`) + - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`) + - `direction` (type `u32`, sqltype `INTEGER`) + - `public` (type `boolean`, sqltype `INTEGER`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `message_flags` (type `u8`, sqltype `INTEGER`) + - `channel_flags` (type `u8`, sqltype `INTEGER`) + - `active` (type `boolean`, sqltype `INTEGER`) + - `last_update` (type `u32`, sqltype `INTEGER`) + - `base_fee_millisatoshi` (type `u32`, sqltype `INTEGER`) + - `fee_per_millionth` (type `u32`, sqltype `INTEGER`) + - `delay` (type `u32`, sqltype `INTEGER`) + - `htlc_minimum_msat` (type `msat`, sqltype `INTEGER`) + - `htlc_maximum_msat` (type `msat`, sqltype `INTEGER`) + - `features` (type `hex`, sqltype `BLOB`) + +- `forwards` indexed by `in_channel and in_htlc_id` (see lightning-listforwards(7)) + - `in_channel` (type `short_channel_id`, sqltype `TEXT`) + - `in_htlc_id` (type `u64`, sqltype `INTEGER`) + - `in_msat` (type `msat`, sqltype `INTEGER`) + - `status` (type `string`, sqltype `TEXT`) + - `received_time` (type `number`, sqltype `REAL`) + - `out_channel` (type `short_channel_id`, sqltype `TEXT`) + - `out_htlc_id` (type `u64`, sqltype `INTEGER`) + - `style` (type `string`, sqltype `TEXT`) + - `fee_msat` (type `msat`, sqltype `INTEGER`) + - `out_msat` (type `msat`, sqltype `INTEGER`) + - `resolved_time` (type `number`, sqltype `REAL`) + - `failcode` (type `u32`, sqltype `INTEGER`) + - `failreason` (type `string`, sqltype `TEXT`) + +- `htlcs` indexed by `short_channel_id and id` (see lightning-listhtlcs(7)) + - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`) + - `id` (type `u64`, sqltype `INTEGER`) + - `expiry` (type `u32`, sqltype `INTEGER`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `direction` (type `string`, sqltype `TEXT`) + - `payment_hash` (type `hash`, sqltype `BLOB`) + - `state` (type `string`, sqltype `TEXT`) + +- `invoices` indexed by `payment_hash` (see lightning-listinvoices(7)) + - `label` (type `string`, sqltype `TEXT`) + - `description` (type `string`, sqltype `TEXT`) + - `payment_hash` (type `hash`, sqltype `BLOB`) + - `status` (type `string`, sqltype `TEXT`) + - `expires_at` (type `u64`, sqltype `INTEGER`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `bolt11` (type `string`, sqltype `TEXT`) + - `bolt12` (type `string`, sqltype `TEXT`) + - `local_offer_id` (type `hash`, sqltype `BLOB`) + - `invreq_payer_note` (type `string`, sqltype `TEXT`) + - `pay_index` (type `u64`, sqltype `INTEGER`) + - `amount_received_msat` (type `msat`, sqltype `INTEGER`) + - `paid_at` (type `u64`, sqltype `INTEGER`) + - `payment_preimage` (type `secret`, sqltype `BLOB`) + +- `nodes` indexed by `nodeid` (see lightning-listnodes(7)) + - `nodeid` (type `pubkey`, sqltype `BLOB`) + - `last_timestamp` (type `u32`, sqltype `INTEGER`) + - `alias` (type `string`, sqltype `TEXT`) + - `color` (type `hex`, sqltype `BLOB`) + - `features` (type `hex`, sqltype `BLOB`) + - related table `nodes_addresses` + - `row` (reference to `nodes.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `type` (type `string`, sqltype `TEXT`) + - `port` (type `u16`, sqltype `INTEGER`) + - `address` (type `string`, sqltype `TEXT`) + - `option_will_fund_lease_fee_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `option_will_fund`) + - `option_will_fund_lease_fee_basis` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`) + - `option_will_fund_funding_weight` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`) + - `option_will_fund_channel_fee_max_base_msat` (type `msat`, sqltype `INTEGER`, from JSON object `option_will_fund`) + - `option_will_fund_channel_fee_max_proportional_thousandths` (type `u32`, sqltype `INTEGER`, from JSON object `option_will_fund`) + - `option_will_fund_compact_lease` (type `hex`, sqltype `BLOB`, from JSON object `option_will_fund`) + +- `offers` indexed by `offer_id` (see lightning-listoffers(7)) + - `offer_id` (type `hash`, sqltype `BLOB`) + - `active` (type `boolean`, sqltype `INTEGER`) + - `single_use` (type `boolean`, sqltype `INTEGER`) + - `bolt12` (type `string`, sqltype `TEXT`) + - `used` (type `boolean`, sqltype `INTEGER`) + - `label` (type `string`, sqltype `TEXT`) + +- `peerchannels` indexed by `peer_id` (see lightning-listpeerchannels(7)) + - `peer_id` (type `pubkey`, sqltype `BLOB`) + - `peer_connected` (type `boolean`, sqltype `INTEGER`) + - `state` (type `string`, sqltype `TEXT`) + - `scratch_txid` (type `txid`, sqltype `BLOB`) + - `feerate_perkw` (type `u32`, sqltype `INTEGER`, from JSON object `feerate`) + - `feerate_perkb` (type `u32`, sqltype `INTEGER`, from JSON object `feerate`) + - `owner` (type `string`, sqltype `TEXT`) + - `short_channel_id` (type `short_channel_id`, sqltype `TEXT`) + - `channel_id` (type `hash`, sqltype `BLOB`) + - `funding_txid` (type `txid`, sqltype `BLOB`) + - `funding_outnum` (type `u32`, sqltype `INTEGER`) + - `initial_feerate` (type `string`, sqltype `TEXT`) + - `last_feerate` (type `string`, sqltype `TEXT`) + - `next_feerate` (type `string`, sqltype `TEXT`) + - `next_fee_step` (type `u32`, sqltype `INTEGER`) + - related table `peerchannels_inflight` + - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `funding_txid` (type `txid`, sqltype `BLOB`) + - `funding_outnum` (type `u32`, sqltype `INTEGER`) + - `feerate` (type `string`, sqltype `TEXT`) + - `total_funding_msat` (type `msat`, sqltype `INTEGER`) + - `our_funding_msat` (type `msat`, sqltype `INTEGER`) + - `scratch_txid` (type `txid`, sqltype `BLOB`) + - `close_to` (type `hex`, sqltype `BLOB`) + - `private` (type `boolean`, sqltype `INTEGER`) + - `opener` (type `string`, sqltype `TEXT`) + - `closer` (type `string`, sqltype `TEXT`) + - related table `peerchannels_features` + - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `features` (type `string`, sqltype `TEXT`) + - `funding_pushed_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`) + - `funding_local_funds_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`) + - `funding_remote_funds_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`) + - `funding_fee_paid_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`) + - `funding_fee_rcvd_msat` (type `msat`, sqltype `INTEGER`, from JSON object `funding`) + - `to_us_msat` (type `msat`, sqltype `INTEGER`) + - `min_to_us_msat` (type `msat`, sqltype `INTEGER`) + - `max_to_us_msat` (type `msat`, sqltype `INTEGER`) + - `total_msat` (type `msat`, sqltype `INTEGER`) + - `fee_base_msat` (type `msat`, sqltype `INTEGER`) + - `fee_proportional_millionths` (type `u32`, sqltype `INTEGER`) + - `dust_limit_msat` (type `msat`, sqltype `INTEGER`) + - `max_total_htlc_in_msat` (type `msat`, sqltype `INTEGER`) + - `their_reserve_msat` (type `msat`, sqltype `INTEGER`) + - `our_reserve_msat` (type `msat`, sqltype `INTEGER`) + - `spendable_msat` (type `msat`, sqltype `INTEGER`) + - `receivable_msat` (type `msat`, sqltype `INTEGER`) + - `minimum_htlc_in_msat` (type `msat`, sqltype `INTEGER`) + - `minimum_htlc_out_msat` (type `msat`, sqltype `INTEGER`) + - `maximum_htlc_out_msat` (type `msat`, sqltype `INTEGER`) + - `their_to_self_delay` (type `u32`, sqltype `INTEGER`) + - `our_to_self_delay` (type `u32`, sqltype `INTEGER`) + - `max_accepted_htlcs` (type `u32`, sqltype `INTEGER`) + - `alias_local` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`) + - `alias_remote` (type `short_channel_id`, sqltype `TEXT`, from JSON object `alias`) + - related table `peerchannels_state_changes` + - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `timestamp` (type `string`, sqltype `TEXT`) + - `old_state` (type `string`, sqltype `TEXT`) + - `new_state` (type `string`, sqltype `TEXT`) + - `cause` (type `string`, sqltype `TEXT`) + - `message` (type `string`, sqltype `TEXT`) + - related table `peerchannels_status` + - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `status` (type `string`, sqltype `TEXT`) + - `in_payments_offered` (type `u64`, sqltype `INTEGER`) + - `in_offered_msat` (type `msat`, sqltype `INTEGER`) + - `in_payments_fulfilled` (type `u64`, sqltype `INTEGER`) + - `in_fulfilled_msat` (type `msat`, sqltype `INTEGER`) + - `out_payments_offered` (type `u64`, sqltype `INTEGER`) + - `out_offered_msat` (type `msat`, sqltype `INTEGER`) + - `out_payments_fulfilled` (type `u64`, sqltype `INTEGER`) + - `out_fulfilled_msat` (type `msat`, sqltype `INTEGER`) + - related table `peerchannels_htlcs` + - `row` (reference to `peerchannels.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `direction` (type `string`, sqltype `TEXT`) + - `id` (type `u64`, sqltype `INTEGER`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `expiry` (type `u32`, sqltype `INTEGER`) + - `payment_hash` (type `hash`, sqltype `BLOB`) + - `local_trimmed` (type `boolean`, sqltype `INTEGER`) + - `status` (type `string`, sqltype `TEXT`) + - `state` (type `string`, sqltype `TEXT`) + - `close_to_addr` (type `string`, sqltype `TEXT`) + - `last_tx_fee_msat` (type `msat`, sqltype `INTEGER`) + - `direction` (type `u32`, sqltype `INTEGER`) + +- `peers` indexed by `id` (see lightning-listpeers(7)) + - `id` (type `pubkey`, sqltype `BLOB`) + - `connected` (type `boolean`, sqltype `INTEGER`) + - related table `peers_netaddr` + - `row` (reference to `peers.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `netaddr` (type `string`, sqltype `TEXT`) + - `remote_addr` (type `string`, sqltype `TEXT`) + - `features` (type `hex`, sqltype `BLOB`) + +- `sendpays` indexed by `payment_hash` (see lightning-listsendpays(7)) + - `id` (type `u64`, sqltype `INTEGER`) + - `groupid` (type `u64`, sqltype `INTEGER`) + - `partid` (type `u64`, sqltype `INTEGER`) + - `payment_hash` (type `hash`, sqltype `BLOB`) + - `status` (type `string`, sqltype `TEXT`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `destination` (type `pubkey`, sqltype `BLOB`) + - `created_at` (type `u64`, sqltype `INTEGER`) + - `amount_sent_msat` (type `msat`, sqltype `INTEGER`) + - `label` (type `string`, sqltype `TEXT`) + - `bolt11` (type `string`, sqltype `TEXT`) + - `description` (type `string`, sqltype `TEXT`) + - `bolt12` (type `string`, sqltype `TEXT`) + - `payment_preimage` (type `secret`, sqltype `BLOB`) + - `erroronion` (type `hex`, sqltype `BLOB`) + +- `transactions` indexed by `hash` (see lightning-listtransactions(7)) + - `hash` (type `txid`, sqltype `BLOB`) + - `rawtx` (type `hex`, sqltype `BLOB`) + - `blockheight` (type `u32`, sqltype `INTEGER`) + - `txindex` (type `u32`, sqltype `INTEGER`) + - `locktime` (type `u32`, sqltype `INTEGER`) + - `version` (type `u32`, sqltype `INTEGER`) + - related table `transactions_inputs` + - `row` (reference to `transactions.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `txid` (type `txid`, sqltype `BLOB`) + - `idx` (type `u32`, sqltype `INTEGER`, from JSON field `index`) + - `sequence` (type `u32`, sqltype `INTEGER`) + - `type` (type `string`, sqltype `TEXT`) + - `channel` (type `short_channel_id`, sqltype `TEXT`) + - related table `transactions_outputs` + - `row` (reference to `transactions.rowid`, sqltype `INTEGER`) + - `arrindex` (index within array, sqltype `INTEGER`) + - `idx` (type `u32`, sqltype `INTEGER`, from JSON field `index`) + - `amount_msat` (type `msat`, sqltype `INTEGER`) + - `scriptPubKey` (type `hex`, sqltype `BLOB`) + - `type` (type `string`, sqltype `TEXT`) + - `channel` (type `short_channel_id`, sqltype `TEXT`) + +[comment]: # (GENERATE-DOC-END) + +RETURN VALUE +------------ + +[comment]: # (FIXME: we don't handle this schema in fromschema.py) +On success, an object containing **rows** is returned. It is an array. Each array entry contains an array of values, each an integer, real number, string or *null*, depending on the sqlite3 type. + +The object may contain **warning\_db\_failure** if the database fails partway through its operation. + +On failure, an error is returned. + +AUTHOR +------ + +Rusty Russell <> is mainly responsible. + +SEE ALSO +-------- + +lightning-listtransactions(7), lightning-listchannels(7), lightning-listpeers(7), lightning-listnodes(7), lightning-listforwards(7). + +RESOURCES +--------- + +Main web site: +[comment]: # ( SHA256STAMP:93309f8c45ea3aa153bfd8822f2748c1254812d41a408de39bacefa292e11374) diff --git a/doc/schemas/sql.request.json b/doc/schemas/sql.request.json new file mode 100644 index 000000000000..8664d15115f6 --- /dev/null +++ b/doc/schemas/sql.request.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": [ + "query" + ], + "properties": { + "query": { + "type": "string" + } + } +} diff --git a/doc/schemas/sql.schema.json b/doc/schemas/sql.schema.json new file mode 100644 index 000000000000..fe249edb5ef6 --- /dev/null +++ b/doc/schemas/sql.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "rows" + ], + "properties": { + "rows": { + "type": "array", + "items": { + "type": "array" + } + }, + "warning_db_failure": { + "type": "string", + "description": "A message if the database encounters an error partway through" + } + } +} diff --git a/plugins/sql.c b/plugins/sql.c index b03266bfd8f9..46e854cebf24 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -27,7 +27,6 @@ static const char schemas[] = /* TODO: * 2. Refresh time in API. - * 5. documentation. * 6. test on mainnet. * 7. Some cool query for documentation. * 8. time_msec fields. From 212c85963503e0fc8f2a3c86c529907a75ab92a6 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:18 +1030 Subject: [PATCH 32/36] plugins/sql: allow some simple functions. And document them! Signed-off-by: Rusty Russell --- doc/lightning-sql.7.md | 22 ++++++++++++++++++++++ plugins/sql.c | 32 ++++++++++++++++++++++++++++++++ tests/test_plugin.py | 33 ++++++++++++++++++++++++++++++--- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/doc/lightning-sql.7.md b/doc/lightning-sql.7.md index bb645244acc9..0b5c7d50a6ba 100644 --- a/doc/lightning-sql.7.md +++ b/doc/lightning-sql.7.md @@ -53,6 +53,28 @@ represented as an integer in the database, so a query will return 0 or * JSON: string * sqlite3: TEXT +PERMITTED SQLITE3 FUNCTIONS +--------------------------- +Writing to the database is not permitted, and limits are placed +on various other query parameters. + +Additionally, only the following functions are allowed: + +* abs +* avg +* coalesce +* count +* hex +* quote +* length +* like +* lower +* upper +* min +* max +* sum +* total + TABLES ------ [comment]: # (GENERATE-DOC-START) diff --git a/plugins/sql.c b/plugins/sql.c index 46e854cebf24..9890bac61b6b 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -264,6 +264,38 @@ static int sqlite_authorize(void *dbq_, int code, return SQLITE_OK; } + /* Some functions are fairly necessary: */ + if (code == SQLITE_FUNCTION) { + if (streq(b, "abs")) + return SQLITE_OK; + if (streq(b, "avg")) + return SQLITE_OK; + if (streq(b, "coalesce")) + return SQLITE_OK; + if (streq(b, "count")) + return SQLITE_OK; + if (streq(b, "hex")) + return SQLITE_OK; + if (streq(b, "quote")) + return SQLITE_OK; + if (streq(b, "length")) + return SQLITE_OK; + if (streq(b, "like")) + return SQLITE_OK; + if (streq(b, "lower")) + return SQLITE_OK; + if (streq(b, "upper")) + return SQLITE_OK; + if (streq(b, "min")) + return SQLITE_OK; + if (streq(b, "max")) + return SQLITE_OK; + if (streq(b, "sum")) + return SQLITE_OK; + if (streq(b, "total")) + return SQLITE_OK; + } + /* See https://www.sqlite.org/c3ref/c_alter_table.html to decode these! */ dbq->authfail = tal_fmt(dbq, "Unauthorized: %u arg1=%s arg2=%s dbname=%s caller=%s", code, diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 237771fbce85..b163f560ffa1 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3280,10 +3280,16 @@ def test_block_added_notifications(node_factory, bitcoind): @pytest.mark.openchannel('v2') @pytest.mark.developer("wants dev-announce-localhost so we see listnodes.addresses") def test_sql(node_factory, bitcoind): + opts = {'experimental-offers': None, + 'dev-allow-localhost': None} + l2opts = {'lease-fee-basis': 50, + 'lease-fee-base-sat': '2000msat', + 'channel-fee-max-base-msat': '500sat', + 'channel-fee-max-proportional-thousandths': 200, + 'sqlfilename': 'sql.sqlite3'} + l2opts.update(opts) l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True, - opts={'experimental-offers': None, - 'sqlfilename': 'sql.sqlite3', - 'dev-allow-localhost': None}) + opts=[opts, l2opts, opts]) ret = l2.rpc.sql("SELECT * FROM forwards;") assert ret == {'rows': []} @@ -3794,6 +3800,27 @@ def test_sql(node_factory, bitcoind): with pytest.raises(RpcError, match='query failed with no such table: peers_channels'): l2.rpc.sql("SELECT * FROM peers_channels;") + # Test subobject case (option_will_fund) + ret = l2.rpc.sql("SELECT option_will_fund_lease_fee_base_msat," + " option_will_fund_lease_fee_basis," + " option_will_fund_funding_weight," + " option_will_fund_channel_fee_max_base_msat," + " option_will_fund_channel_fee_max_proportional_thousandths," + " option_will_fund_compact_lease" + " FROM nodes WHERE HEX(nodeid) = '{}';".format(l2.info['id'].upper())) + optret = only_one(l2.rpc.listnodes(l2.info['id'])['nodes'])['option_will_fund'] + row = only_one(ret['rows']) + assert row == [v for v in optret.values()] + + # Correctly handles missing object. + assert l2.rpc.sql("SELECT option_will_fund_lease_fee_base_msat," + " option_will_fund_lease_fee_basis," + " option_will_fund_funding_weight," + " option_will_fund_channel_fee_max_base_msat," + " option_will_fund_channel_fee_max_proportional_thousandths," + " option_will_fund_compact_lease" + " FROM nodes WHERE HEX(nodeid) = '{}';".format(l1.info['id'].upper())) == {'rows': [[None] * 6]} + def test_sql_deprecated(node_factory, bitcoind): # deprecated-apis breaks schemas... From 8d60994ca06ea6ffb3663f6ea66d487aee390be4 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:18 +1030 Subject: [PATCH 33/36] plugins/sql: add bkpr-listaccountevents and bkpr-listincome support. This *would* be a 1-line change (add it to Makefile) except that we previously assumed a "list" prefix on commands. These use the default refreshing, but they could be done better using the time-range parameters. Suggested-by: @niftynei Signed-off-by: Rusty Russell --- doc/lightning-sql.7.md | 32 +++++++++++++++++++++++- plugins/Makefile | 2 +- tests/test_plugin.py | 56 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/doc/lightning-sql.7.md b/doc/lightning-sql.7.md index 0b5c7d50a6ba..f682a00e77ec 100644 --- a/doc/lightning-sql.7.md +++ b/doc/lightning-sql.7.md @@ -79,6 +79,36 @@ TABLES ------ [comment]: # (GENERATE-DOC-START) The following tables are currently supported: +- `bkpr_accountevents` (see lightning-bkpr-listaccountevents(7)) + - `account` (type `string`, sqltype `TEXT`) + - `type` (type `string`, sqltype `TEXT`) + - `tag` (type `string`, sqltype `TEXT`) + - `credit_msat` (type `msat`, sqltype `INTEGER`) + - `debit_msat` (type `msat`, sqltype `INTEGER`) + - `currency` (type `string`, sqltype `TEXT`) + - `timestamp` (type `u32`, sqltype `INTEGER`) + - `outpoint` (type `string`, sqltype `TEXT`) + - `blockheight` (type `u32`, sqltype `INTEGER`) + - `origin` (type `string`, sqltype `TEXT`) + - `payment_id` (type `hex`, sqltype `BLOB`) + - `txid` (type `txid`, sqltype `BLOB`) + - `description` (type `string`, sqltype `TEXT`) + - `fees_msat` (type `msat`, sqltype `INTEGER`) + - `is_rebalance` (type `boolean`, sqltype `INTEGER`) + - `part_id` (type `u32`, sqltype `INTEGER`) + +- `bkpr_income` (see lightning-bkpr-listincome(7)) + - `account` (type `string`, sqltype `TEXT`) + - `tag` (type `string`, sqltype `TEXT`) + - `credit_msat` (type `msat`, sqltype `INTEGER`) + - `debit_msat` (type `msat`, sqltype `INTEGER`) + - `currency` (type `string`, sqltype `TEXT`) + - `timestamp` (type `u32`, sqltype `INTEGER`) + - `description` (type `string`, sqltype `TEXT`) + - `outpoint` (type `string`, sqltype `TEXT`) + - `txid` (type `txid`, sqltype `BLOB`) + - `payment_id` (type `hex`, sqltype `BLOB`) + - `channels` indexed by `short_channel_id` (see lightning-listchannels(7)) - `source` (type `pubkey`, sqltype `BLOB`) - `destination` (type `pubkey`, sqltype `BLOB`) @@ -334,4 +364,4 @@ RESOURCES --------- Main web site: -[comment]: # ( SHA256STAMP:93309f8c45ea3aa153bfd8822f2748c1254812d41a408de39bacefa292e11374) +[comment]: # ( SHA256STAMP:dbb9286cf31dc82b33143d5274b1c4eecc75c5ba1dfc18bdf21b4baab585bd45) diff --git a/plugins/Makefile b/plugins/Makefile index fad17d248b74..e02db26d33e3 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -205,7 +205,7 @@ plugins/fetchinvoice: $(PLUGIN_FETCHINVOICE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_CO plugins/funder: bitcoin/psbt.o common/psbt_open.o $(PLUGIN_FUNDER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) # This covers all the low-level list RPCs which return simple arrays -SQL_LISTRPCS := listchannels listforwards listhtlcs listinvoices listnodes listoffers listpeers listpeerchannels listtransactions listsendpays +SQL_LISTRPCS := listchannels listforwards listhtlcs listinvoices listnodes listoffers listpeers listpeerchannels listtransactions listsendpays bkpr-listaccountevents bkpr-listincome SQL_LISTRPCS_SCHEMAS := $(foreach l,$(SQL_LISTRPCS),doc/schemas/$l.schema.json) # We squeeze: # descriptions (we don't need) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index b163f560ffa1..6a6b43a2c885 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3727,7 +3727,61 @@ def test_sql(node_factory, bitcoind): {'name': 'type', 'type': 'string'}, {'name': 'channel', - 'type': 'short_channel_id'}]}} + 'type': 'short_channel_id'}]}, + 'bkpr_accountevents': { + 'columns': [{'name': 'account', + 'type': 'string'}, + {'name': 'type', + 'type': 'string'}, + {'name': 'tag', + 'type': 'string'}, + {'name': 'credit_msat', + 'type': 'msat'}, + {'name': 'debit_msat', + 'type': 'msat'}, + {'name': 'currency', + 'type': 'string'}, + {'name': 'timestamp', + 'type': 'u32'}, + {'name': 'outpoint', + 'type': 'string'}, + {'name': 'blockheight', + 'type': 'u32'}, + {'name': 'origin', + 'type': 'string'}, + {'name': 'payment_id', + 'type': 'hex'}, + {'name': 'txid', + 'type': 'txid'}, + {'name': 'description', + 'type': 'string'}, + {'name': 'fees_msat', + 'type': 'msat'}, + {'name': 'is_rebalance', + 'type': 'boolean'}, + {'name': 'part_id', + 'type': 'u32'}]}, + 'bkpr_income': { + 'columns': [{'name': 'account', + 'type': 'string'}, + {'name': 'tag', + 'type': 'string'}, + {'name': 'credit_msat', + 'type': 'msat'}, + {'name': 'debit_msat', + 'type': 'msat'}, + {'name': 'currency', + 'type': 'string'}, + {'name': 'timestamp', + 'type': 'u32'}, + {'name': 'description', + 'type': 'string'}, + {'name': 'outpoint', + 'type': 'string'}, + {'name': 'txid', + 'type': 'txid'}, + {'name': 'payment_id', + 'type': 'hex'}]}} # Very rough checks of other list commands (make sure l2 has one of each) l2.rpc.offer(1, 'desc') From 28cf990c07490bb2688f4ac3766d2ade59dad2ca Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:18 +1030 Subject: [PATCH 34/36] plugins/sql: listsqlschemas command to retrieve schemas. Good for detection of what fields are present. Signed-off-by: Rusty Russell --- doc/Makefile | 1 + doc/index.rst | 1 + doc/lightning-listsqlschemas.7.md | 109 +++++++++++++++++++++++ doc/lightning-sql.7.md | 3 + doc/schemas/listsqlschemas.request.json | 10 +++ doc/schemas/listsqlschemas.schema.json | 67 ++++++++++++++ plugins/sql.c | 112 ++++++++++++++++++++++++ tests/test_plugin.py | 28 ++++++ 8 files changed, 331 insertions(+) create mode 100644 doc/lightning-listsqlschemas.7.md create mode 100644 doc/schemas/listsqlschemas.request.json create mode 100644 doc/schemas/listsqlschemas.schema.json diff --git a/doc/Makefile b/doc/Makefile index 174a94131f0d..764e5c890fdc 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -60,6 +60,7 @@ MANPAGES := doc/lightning-cli.1 \ doc/lightning-listpeers.7 \ doc/lightning-listpeerchannels.7 \ doc/lightning-listsendpays.7 \ + doc/lightning-listsqlschemas.7 \ doc/lightning-makesecret.7 \ doc/lightning-multifundchannel.7 \ doc/lightning-multiwithdraw.7 \ diff --git a/doc/index.rst b/doc/index.rst index 822679c109aa..2f05acd7f7e2 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -88,6 +88,7 @@ Core Lightning Documentation lightning-listpeerchannels lightning-listpeers lightning-listsendpays + lightning-listsqlschemas lightning-listtransactions lightning-makesecret lightning-multifundchannel diff --git a/doc/lightning-listsqlschemas.7.md b/doc/lightning-listsqlschemas.7.md new file mode 100644 index 000000000000..13e725fce47f --- /dev/null +++ b/doc/lightning-listsqlschemas.7.md @@ -0,0 +1,109 @@ +lightning-listsqlschemas -- Command to example lightning-sql schemas +==================================================================== + +SYNOPSIS +-------- + +**listsqlschemas** [*table*] + +DESCRIPTION +----------- + +This allows you to examine the schemas at runtime; while they are fully +documented for the current release in lightning-sql(7), as fields are +added or deprecated, you can use this command to determine what fields +are present. + +If *table* is given, only that table is in the resulting list, otherwise +all tables are listed. + +EXAMPLE JSON REQUEST +------------ +```json +{ + "id": 82, + "method": "listsqlschemas", + "params": { + "table": "offers" + } +} +``` + +EXAMPLE JSON RESPONSE +----- +```json +{ + "schemas": [ + { + "tablename": "offers", + "columns": [ + { + "name": "offer_id", + "type": "BLOB" + }, + { + "name": "active", + "type": "INTEGER" + }, + { + "name": "single_use", + "type": "INTEGER" + }, + { + "name": "bolt12", + "type": "TEXT" + }, + { + "name": "bolt12_unsigned", + "type": "TEXT" + }, + { + "name": "used", + "type": "INTEGER" + }, + { + "name": "label", + "type": "TEXT" + } + ], + "indices": [ + [ + "offer_id" + ] + ] + } + ] +} +``` + +RETURN VALUE +------------ + +[comment]: # (GENERATE-FROM-SCHEMA-START) +On success, an object containing **schemas** is returned. It is an array of objects, where each object contains: + +- **tablename** (string): the name of the table +- **columns** (array of objects): the columns, in database order: + - **name** (string): the name the column + - **type** (string): the SQL type of the column (one of "INTEGER", "BLOB", "TEXT", "REAL") +- **indices** (array of arrays, optional): Any index we created to speed lookups: + - The columns for this index: + - The column name + +[comment]: # (GENERATE-FROM-SCHEMA-END) + +AUTHOR +------ + +Rusty Russell <> is mainly responsible. + +SEE ALSO +-------- + +lightning-sql(7). + +RESOURCES +--------- + +Main web site: +[comment]: # ( SHA256STAMP:3ac985dd8ef6959b327e6e6a79079db3ad51423bc4e469799a12ae74b2e75697) diff --git a/doc/lightning-sql.7.md b/doc/lightning-sql.7.md index f682a00e77ec..480ca77388f0 100644 --- a/doc/lightning-sql.7.md +++ b/doc/lightning-sql.7.md @@ -19,6 +19,9 @@ cache `listnodes` and `listchannels`) which then processes the results. It is, however faster for remote access if the result of the query is much smaller than the list commands would be. +Note that queries like "SELECT *" are fragile, as columns will +change across releases; see lightning-listsqlschemas(7). + TREATMENT OF TYPES ------------------ diff --git a/doc/schemas/listsqlschemas.request.json b/doc/schemas/listsqlschemas.request.json new file mode 100644 index 000000000000..f12785b1b74c --- /dev/null +++ b/doc/schemas/listsqlschemas.request.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": [], + "properties": { + "table": { + "type": "string" + } + } +} diff --git a/doc/schemas/listsqlschemas.schema.json b/doc/schemas/listsqlschemas.schema.json new file mode 100644 index 000000000000..01143f1b8554 --- /dev/null +++ b/doc/schemas/listsqlschemas.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "schemas" + ], + "properties": { + "schemas": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "tablename", + "columns" + ], + "properties": { + "tablename": { + "type": "string", + "description": "the name of the table" + }, + "columns": { + "type": "array", + "description": "the columns, in database order", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type" + ], + "properties": { + "name": { + "type": "string", + "description": "the name the column" + }, + "type": { + "type": "string", + "enum": [ + "INTEGER", + "BLOB", + "TEXT", + "REAL" + ], + "description": "the SQL type of the column" + } + } + } + }, + "indices": { + "type": "array", + "description": "Any index we created to speed lookups", + "items": { + "type": "array", + "description": "The columns for this index", + "items": { + "type": "string", + "description": "The column name" + } + } + } + } + } + } + } +} diff --git a/plugins/sql.c b/plugins/sql.c index 9890bac61b6b..bf870d5613ae 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1001,6 +1001,111 @@ static bool ignore_column(const struct table_desc *td, const jsmntok_t *t) return false; } +static struct command_result *param_tablename(struct command *cmd, + const char *name, + const char *buffer, + const jsmntok_t *tok, + struct table_desc **td) +{ + *td = strmap_getn(&tablemap, buffer + tok->start, + tok->end - tok->start); + if (!*td) + return command_fail_badparam(cmd, name, buffer, tok, + "Unknown table"); + return NULL; +} + +static void json_add_column(struct json_stream *js, + const char *dbname, + const char *sqltypename) +{ + json_object_start(js, NULL); + json_add_string(js, "name", dbname); + json_add_string(js, "type", sqltypename); + json_object_end(js); +} + +static void json_add_columns(struct json_stream *js, + const struct table_desc *td) +{ + for (size_t i = 0; i < tal_count(td->columns); i++) { + if (td->columns[i].sub) { + if (td->columns[i].sub->is_subobject) + json_add_columns(js, td->columns[i].sub); + continue; + } + json_add_column(js, td->columns[i].dbname, + fieldtypemap[td->columns[i].ftype].sqltype); + } +} + +static void json_add_schema(struct json_stream *js, + const struct table_desc *td) +{ + bool have_indices; + + json_object_start(js, NULL); + json_add_string(js, "tablename", td->name); + /* This needs to be an array, not a dictionary, since dicts + * are often treated as unordered, and order is critical! */ + json_array_start(js, "columns"); + if (td->parent) { + json_add_column(js, "row", "INTEGER"); + json_add_column(js, "arrindex", "INTEGER"); + } + json_add_columns(js, td); + json_array_end(js); + + /* Don't print indices entry unless we have an index! */ + have_indices = false; + for (size_t i = 0; i < ARRAY_SIZE(indices); i++) { + if (!streq(indices[i].tablename, td->name)) + continue; + if (!have_indices) { + json_array_start(js, "indices"); + have_indices = true; + } + json_array_start(js, NULL); + for (size_t j = 0; j < ARRAY_SIZE(indices[i].fields); j++) { + if (indices[i].fields[j]) + json_add_string(js, NULL, indices[i].fields[j]); + } + json_array_end(js); + } + if (have_indices) + json_array_end(js); + json_object_end(js); +} + +static bool add_one_schema(const char *member, struct table_desc *td, + struct json_stream *js) +{ + json_add_schema(js, td); + return true; +} + +static struct command_result *json_listsqlschemas(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct table_desc *td; + struct json_stream *ret; + + if (!param(cmd, buffer, params, + p_opt("table", param_tablename, &td), + NULL)) + return command_param_failed(); + + ret = jsonrpc_stream_success(cmd); + json_array_start(ret, "schemas"); + if (td) + json_add_schema(ret, td); + else + strmap_iterate(&tablemap, add_one_schema, ret); + json_array_end(ret); + return command_finished(cmd, ret); +} + /* Creates sql statements, initializes table */ static void finish_td(struct plugin *plugin, struct table_desc *td) { @@ -1353,6 +1458,13 @@ static const struct plugin_command commands[] = { { "This is the greatest plugin command ever!", json_sql, }, + { + "listsqlschemas", + "misc", + "Display schemas for internal sql tables, or just {table}", + "This is the greatest plugin command ever!", + json_listsqlschemas, + }, }; static const char *fmt_indexes(const tal_t *ctx, const char *table) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 6a6b43a2c885..2b17978b6099 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -3783,6 +3783,34 @@ def test_sql(node_factory, bitcoind): {'name': 'payment_id', 'type': 'hex'}]}} + sqltypemap = {'string': 'TEXT', + 'boolean': 'INTEGER', + 'u8': 'INTEGER', + 'u16': 'INTEGER', + 'u32': 'INTEGER', + 'u64': 'INTEGER', + 'msat': 'INTEGER', + 'hex': 'BLOB', + 'hash': 'BLOB', + 'txid': 'BLOB', + 'pubkey': 'BLOB', + 'secret': 'BLOB', + 'number': 'REAL', + 'short_channel_id': 'TEXT'} + + # Check schemas match. + for table, schema in expected_schemas.items(): + res = only_one(l2.rpc.listsqlschemas(table)['schemas']) + assert res['tablename'] == table + assert res.get('indices') == schema.get('indices') + sqlcolumns = [{'name': c['name'], 'type': sqltypemap[c['type']]} for c in schema['columns']] + assert res['columns'] == sqlcolumns + + # Make sure we didn't miss any + assert (sorted([s['tablename'] for s in l1.rpc.listsqlschemas()['schemas']]) + == sorted(expected_schemas.keys())) + assert len(l1.rpc.listsqlschemas()['schemas']) == len(expected_schemas) + # Very rough checks of other list commands (make sure l2 has one of each) l2.rpc.offer(1, 'desc') l2.rpc.invoice(1, 'label', 'desc') From cf3f295bf46bf0405d359e46a6248c508be4469c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:18 +1030 Subject: [PATCH 35/36] doc: add examples for sql plugin. Prompted by @shahanafarooqui's playing with examples and finding common errors. Signed-off-by: Rusty Russell --- doc/lightning-sql.7.md | 98 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/doc/lightning-sql.7.md b/doc/lightning-sql.7.md index 480ca77388f0..237bbf02e698 100644 --- a/doc/lightning-sql.7.md +++ b/doc/lightning-sql.7.md @@ -353,6 +353,104 @@ The object may contain **warning\_db\_failure** if the database fails partway th On failure, an error is returned. +EXAMPLES +-------- +Here are some example using lightning-cli. Note that you may need to +use `-o` if you use queries which contain `=` (which make +lightning-cli(1) default to keyword style): + +A simple peer selection query: + +``` +$ lightning-cli sql "SELECT id FROM peers" +{ + "rows": [ + [ + "02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00" + ] + ] +} +``` + +A statement containing using `=` needs `-o`: + +``` +$ lightning-cli sql -o "SELECT node_id,last_timestamp FROM nodes WHERE last_timestamp>=1669578892" +{ + "rows": [ + [ + "02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00", + 1669601603 + ] + ] +} +``` + +If you want to compare a BLOB column, `x'hex'` or `X'hex'` are needed: + +``` +$ lightning-cli sql -o "SELECT nodeid FROM nodes WHERE nodeid != x'03c9d25b6c0ce4bde5ad97d7ab83f00ae8bd3800a98ccbee36f3c3205315147de1';" +{ + "rows": [ + [ + "0214739d625944f8fdc0da9d2ef44dbd7af58443685e494117b51410c5c3ff973a" + ], + [ + "02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00" + ] + ] +} +$ lightning-cli sql -o "SELECT nodeid FROM nodes WHERE nodeid IN (x'03c9d25b6c0ce4bde5ad97d7ab83f00ae8bd3800a98ccbee36f3c3205315147de1', x'02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00')" +{ + "rows": [ + [ + "02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00" + ], + [ + "03c9d25b6c0ce4bde5ad97d7ab83f00ae8bd3800a98ccbee36f3c3205315147de1" + ] + ] +} +``` + +Related tables are usually referenced by JOIN: + +``` +$ lightning-cli sql -o "SELECT nodeid, alias, nodes_addresses.type, nodes_addresses.port, nodes_addresses.address FROM nodes INNER JOIN nodes_addresses ON nodes_addresses.row = nodes.rowid" +{ + "rows": [ + [ + "02ba9965e3db660385bd1dd2c09dd032e0f2179a94fc5db8917b60adf0b363da00", + "YELLOWWATCH-22.11rc2-31-gcd7593b", + "dns", + 7272, + "localhost" + ], + [ + "0214739d625944f8fdc0da9d2ef44dbd7af58443685e494117b51410c5c3ff973a", + "HOPPINGSQUIRREL-1rc2-31-gcd7593b", + "dns", + 7171, + "localhost" + ] + ] +} +``` + +Simple function usage, in this case COUNT. Strings inside arrays need +", and ' to protect them from the shell: + +``` +$ lightning-cli sql 'SELECT COUNT(*) FROM nodes" +{ + "rows": [ + [ + 3 + ] + ] +} +``` + AUTHOR ------ From c2face414b92d7f1349168a38a90a9f233dc9c16 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Jan 2023 16:54:18 +1030 Subject: [PATCH 36/36] typo fixes found by @niftynei Also, put the "added" lines in the request schemas for new commands: this doesn't do anything (yet?) but it keeps `make schema-added-check` happy. Signed-off-by: Rusty Russell --- common/gossip_store.h | 2 +- doc/lightning-listsqlschemas.7.md | 4 ++-- doc/schemas/listsqlschemas.request.json | 1 + doc/schemas/listsqlschemas.schema.json | 2 +- doc/schemas/sql.request.json | 1 + 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/common/gossip_store.h b/common/gossip_store.h index 52bc9d98df56..d57af87fe1fe 100644 --- a/common/gossip_store.h +++ b/common/gossip_store.h @@ -50,7 +50,7 @@ struct gossip_rcvd_filter; */ struct gossip_hdr { beint16_t flags; /* Length of message after header. */ - beint16_t len; /* Length of message after header. */ + beint16_t len; /* GOSSIP_STORE_xxx_BIT flags. */ beint32_t crc; /* crc of message of timestamp, after header. */ beint32_t timestamp; /* timestamp of msg. */ }; diff --git a/doc/lightning-listsqlschemas.7.md b/doc/lightning-listsqlschemas.7.md index 13e725fce47f..52a4480191b6 100644 --- a/doc/lightning-listsqlschemas.7.md +++ b/doc/lightning-listsqlschemas.7.md @@ -84,7 +84,7 @@ On success, an object containing **schemas** is returned. It is an array of obj - **tablename** (string): the name of the table - **columns** (array of objects): the columns, in database order: - - **name** (string): the name the column + - **name** (string): the name of the column - **type** (string): the SQL type of the column (one of "INTEGER", "BLOB", "TEXT", "REAL") - **indices** (array of arrays, optional): Any index we created to speed lookups: - The columns for this index: @@ -106,4 +106,4 @@ RESOURCES --------- Main web site: -[comment]: # ( SHA256STAMP:3ac985dd8ef6959b327e6e6a79079db3ad51423bc4e469799a12ae74b2e75697) +[comment]: # ( SHA256STAMP:29ce2ff3f7cab8a4a90d09fa02fa8176008413272d46c0fe7faa6216f11bb2c6) diff --git a/doc/schemas/listsqlschemas.request.json b/doc/schemas/listsqlschemas.request.json index f12785b1b74c..a1b83e4d867e 100644 --- a/doc/schemas/listsqlschemas.request.json +++ b/doc/schemas/listsqlschemas.request.json @@ -2,6 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "required": [], + "added": "v23.02", "properties": { "table": { "type": "string" diff --git a/doc/schemas/listsqlschemas.schema.json b/doc/schemas/listsqlschemas.schema.json index 01143f1b8554..def50479caac 100644 --- a/doc/schemas/listsqlschemas.schema.json +++ b/doc/schemas/listsqlschemas.schema.json @@ -33,7 +33,7 @@ "properties": { "name": { "type": "string", - "description": "the name the column" + "description": "the name of the column" }, "type": { "type": "string", diff --git a/doc/schemas/sql.request.json b/doc/schemas/sql.request.json index 8664d15115f6..97c6dd25eee7 100644 --- a/doc/schemas/sql.request.json +++ b/doc/schemas/sql.request.json @@ -4,6 +4,7 @@ "required": [ "query" ], + "added": "v23.02", "properties": { "query": { "type": "string"