diff --git a/gossipd/gossipd.c b/gossipd/gossipd.c index 48903d5fb..74e46ad26 100644 --- a/gossipd/gossipd.c +++ b/gossipd/gossipd.c @@ -1947,55 +1947,62 @@ static struct io_plan *getroute_req(struct io_conn *conn, struct daemon *daemon, * marshalling the channel information for all channels into an array of * gossip_getchannels_entry, which lightningd converts to JSON. Each channel * is represented by two half_chan; one in each direction. - * - * FIXME: I run a lightning node permanently under valgrind, and Christian ran - * `listchannels` on it. After about 15 minutes I simply rebooted. There's - * been some optimization since then, but blocking gossipd to marshall all the - * channels will become in issue in future, I expect. We may even hit the - * 2^24 internal message limit. */ -static void append_half_channel(struct gossip_getchannels_entry **entries, - const struct chan *chan, - int idx) +static struct gossip_halfchannel_entry *hc_entry(const tal_t *ctx, + const struct chan *chan, + int idx) { - const struct half_chan *c = &chan->half[idx]; - struct gossip_getchannels_entry e; - - /* If we've never seen a channel_update for this direction... */ - if (!is_halfchan_defined(c)) - return; - /* Our 'struct chan' contains two nodes: they are in pubkey_cmp order * (ie. chan->nodes[0] is the lesser pubkey) and this is the same as * the direction bit in `channel_update`s `channel_flags`. * * The halfchans are arranged so that half[0] src == nodes[0], and we - * use that here. We also avoid using libsecp256k1 to translate the - * pubkeys to DER and back: that proves quite expensive, and we assume - * we're on the same architecture as lightningd, so we just send them - * raw in this case. */ - e.source = chan->nodes[idx]->id; - e.destination = chan->nodes[!idx]->id; - e.sat = chan->sat; - e.channel_flags = c->channel_flags; - e.message_flags = c->message_flags; - e.local_disabled = chan->local_disabled; - e.public = is_chan_public(chan); - e.short_channel_id = chan->scid; - e.last_update_timestamp = c->last_timestamp; - e.base_fee_msat = c->base_fee; - e.fee_per_millionth = c->proportional_fee; - e.delay = c->delay; + * use that here. */ + const struct half_chan *c = &chan->half[idx]; + struct gossip_halfchannel_entry *e; - tal_arr_expand(entries, e); + /* If we've never seen a channel_update for this direction... */ + if (!is_halfchan_defined(c)) + return NULL; + + e = tal(ctx, struct gossip_halfchannel_entry); + e->channel_flags = c->channel_flags; + e->message_flags = c->message_flags; + e->last_update_timestamp = c->last_timestamp; + e->base_fee_msat = c->base_fee; + e->fee_per_millionth = c->proportional_fee; + e->delay = c->delay; + + return e; } -/*~ Marshal (possibly) both channel directions into entries */ -static void append_channel(struct gossip_getchannels_entry **entries, - const struct chan *chan) +/*~ Marshal (possibly) both channel directions into entries. */ +static void append_channel(const struct gossip_getchannels_entry ***entries, + const struct chan *chan, + const struct node_id *srcfilter) { - append_half_channel(entries, chan, 0); - append_half_channel(entries, chan, 1); + struct gossip_getchannels_entry *e = tal(*entries, struct gossip_getchannels_entry); + + e->node[0] = chan->nodes[0]->id; + e->node[1] = chan->nodes[1]->id; + e->sat = chan->sat; + e->local_disabled = chan->local_disabled; + e->public = is_chan_public(chan); + e->short_channel_id = chan->scid; + if (!srcfilter || node_id_eq(&e->node[0], srcfilter)) + e->e[0] = hc_entry(*entries, chan, 0); + else + e->e[0] = NULL; + if (!srcfilter || node_id_eq(&e->node[1], srcfilter)) + e->e[1] = hc_entry(*entries, chan, 1); + else + e->e[1] = NULL; + + /* We choose not to tell lightningd about channels with no updates, + * as they're unusable and can't be represented in the listchannels + * JSON output we use anyway. */ + if (e->e[0] || e->e[1]) + tal_arr_expand(entries, e); } /*~ This is where lightningd asks for all channels we know about. */ @@ -2004,7 +2011,7 @@ static struct io_plan *getchannels_req(struct io_conn *conn, const u8 *msg) { u8 *out; - struct gossip_getchannels_entry *entries; + const struct gossip_getchannels_entry **entries; struct chan *chan; struct short_channel_id *scid; struct node_id *source; @@ -2013,12 +2020,12 @@ static struct io_plan *getchannels_req(struct io_conn *conn, if (!fromwire_gossip_getchannels_request(msg, msg, &scid, &source)) master_badmsg(WIRE_GOSSIP_GETCHANNELS_REQUEST, msg); - entries = tal_arr(tmpctx, struct gossip_getchannels_entry, 0); + entries = tal_arr(tmpctx, const struct gossip_getchannels_entry *, 0); /* They can ask about a particular channel by short_channel_id */ if (scid) { chan = get_channel(daemon->rstate, scid); if (chan) - append_channel(&entries, chan); + append_channel(&entries, chan, NULL); } else if (source) { struct node *s = get_node(daemon->rstate, source); if (s) { @@ -2026,9 +2033,7 @@ static struct io_plan *getchannels_req(struct io_conn *conn, struct chan *c; for (c = first_chan(s, &i); c; c = next_chan(s, &i)) { - append_half_channel(&entries, - c, - !half_chan_to(s, c)); + append_channel(&entries, c, source); } } } else { @@ -2039,7 +2044,7 @@ static struct io_plan *getchannels_req(struct io_conn *conn, for (chan = uintmap_first(&daemon->rstate->chanmap, &idx); chan; chan = uintmap_after(&daemon->rstate->chanmap, &idx)) { - append_channel(&entries, chan); + append_channel(&entries, chan, NULL); } } diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 910a1aced..462e7cbb6 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -367,12 +367,43 @@ static const struct json_command getroute_command = { }; AUTODATA(json_command, &getroute_command); +static void json_add_halfchan(struct json_stream *response, + const struct gossip_getchannels_entry *e, + int idx) +{ + const struct gossip_halfchannel_entry *he = e->e[idx]; + if (!he) + return; + + json_object_start(response, NULL); + json_add_node_id(response, "source", &e->node[idx]); + json_add_node_id(response, "destination", &e->node[!idx]); + json_add_short_channel_id(response, "short_channel_id", + &e->short_channel_id); + json_add_bool(response, "public", e->public); + json_add_amount_sat(response, e->sat, + "satoshis", "amount_msat"); + json_add_num(response, "message_flags", he->message_flags); + json_add_num(response, "channel_flags", he->channel_flags); + /* Prior to spec v0891374d47ddffa64c5a2e6ad151247e3d6b7a59, these two were a single u16 field */ + if (deprecated_apis) + json_add_num(response, "flags", ((u16)he->message_flags << 8) | he->channel_flags); + json_add_bool(response, "active", + !(he->channel_flags & ROUTING_FLAGS_DISABLED) + && !e->local_disabled); + json_add_num(response, "last_update", he->last_update_timestamp); + json_add_num(response, "base_fee_millisatoshi", he->base_fee_msat); + json_add_num(response, "fee_per_millionth", he->fee_per_millionth); + json_add_num(response, "delay", he->delay); + json_object_end(response); +} + /* Called upon receiving a getchannels_reply from `gossipd` */ static void json_listchannels_reply(struct subd *gossip UNUSED, const u8 *reply, const int *fds UNUSED, struct command *cmd) { size_t i; - struct gossip_getchannels_entry *entries; + struct gossip_getchannels_entry **entries; struct json_stream *response; if (!fromwire_gossip_getchannels_reply(reply, reply, &entries)) { @@ -385,33 +416,8 @@ static void json_listchannels_reply(struct subd *gossip UNUSED, const u8 *reply, json_object_start(response, NULL); json_array_start(response, "channels"); for (i = 0; i < tal_count(entries); i++) { - json_object_start(response, NULL); - json_add_node_id(response, "source", - &entries[i].source); - json_add_node_id(response, "destination", - &entries[i].destination); - json_add_string(response, "short_channel_id", - type_to_string(reply, struct short_channel_id, - &entries[i].short_channel_id)); - json_add_bool(response, "public", entries[i].public); - json_add_amount_sat(response, entries[i].sat, - "satoshis", "amount_msat"); - json_add_num(response, "message_flags", entries[i].message_flags); - json_add_num(response, "channel_flags", entries[i].channel_flags); - /* Prior to spec v0891374d47ddffa64c5a2e6ad151247e3d6b7a59, these two were a single u16 field */ - if (deprecated_apis) - json_add_num(response, "flags", ((u16)entries[i].message_flags << 8) | entries[i].channel_flags); - json_add_bool(response, "active", - !(entries[i].channel_flags & ROUTING_FLAGS_DISABLED) - && !entries[i].local_disabled); - json_add_num(response, "last_update", - entries[i].last_update_timestamp); - json_add_num(response, "base_fee_millisatoshi", - entries[i].base_fee_msat); - json_add_num(response, "fee_per_millionth", - entries[i].fee_per_millionth); - json_add_num(response, "delay", entries[i].delay); - json_object_end(response); + json_add_halfchan(response, entries[i], 0); + json_add_halfchan(response, entries[i], 1); } json_array_end(response); json_object_end(response); diff --git a/lightningd/gossip_msg.c b/lightningd/gossip_msg.c index 2b6404606..7150ef9dd 100644 --- a/lightningd/gossip_msg.c +++ b/lightningd/gossip_msg.c @@ -96,38 +96,76 @@ void towire_route_info(u8 **pptr, const struct route_info *entry) towire_u16(pptr, entry->cltv_expiry_delta); } -void fromwire_gossip_getchannels_entry(const u8 **pptr, size_t *max, - struct gossip_getchannels_entry *entry) +static void fromwire_gossip_halfchannel_entry(const u8 **pptr, size_t *max, + struct gossip_halfchannel_entry *entry) { - fromwire_short_channel_id(pptr, max, &entry->short_channel_id); - fromwire_node_id(pptr, max, &entry->source); - fromwire_node_id(pptr, max, &entry->destination); - entry->sat = fromwire_amount_sat(pptr, max); entry->message_flags = fromwire_u8(pptr, max); entry->channel_flags = fromwire_u8(pptr, max); - entry->public = fromwire_bool(pptr, max); - entry->local_disabled = fromwire_bool(pptr, max); entry->last_update_timestamp = fromwire_u32(pptr, max); + entry->delay = fromwire_u32(pptr, max); entry->base_fee_msat = fromwire_u32(pptr, max); entry->fee_per_millionth = fromwire_u32(pptr, max); - entry->delay = fromwire_u32(pptr, max); +} + +struct gossip_getchannels_entry * +fromwire_gossip_getchannels_entry(const tal_t *ctx, + const u8 **pptr, size_t *max) +{ + struct gossip_getchannels_entry *entry; + + entry= tal(ctx, struct gossip_getchannels_entry); + fromwire_node_id(pptr, max, &entry->node[0]); + fromwire_node_id(pptr, max, &entry->node[1]); + entry->sat = fromwire_amount_sat(pptr, max); + fromwire_short_channel_id(pptr, max, &entry->short_channel_id); + entry->public = fromwire_bool(pptr, max); + entry->local_disabled = fromwire_bool(pptr, max); + + if (fromwire_bool(pptr, max)) { + entry->e[0] = tal(entry, struct gossip_halfchannel_entry); + fromwire_gossip_halfchannel_entry(pptr, max, entry->e[0]); + } else + entry->e[0] = NULL; + if (fromwire_bool(pptr, max)) { + entry->e[1] = tal(entry, struct gossip_halfchannel_entry); + fromwire_gossip_halfchannel_entry(pptr, max, entry->e[1]); + } else + entry->e[1] = NULL; + + return entry; +} + +static void towire_gossip_halfchannel_entry(u8 **pptr, + const struct gossip_halfchannel_entry *entry) +{ + towire_u8(pptr, entry->message_flags); + towire_u8(pptr, entry->channel_flags); + towire_u32(pptr, entry->last_update_timestamp); + towire_u32(pptr, entry->delay); + towire_u32(pptr, entry->base_fee_msat); + towire_u32(pptr, entry->fee_per_millionth); } void towire_gossip_getchannels_entry(u8 **pptr, const struct gossip_getchannels_entry *entry) { - towire_short_channel_id(pptr, &entry->short_channel_id); - towire_node_id(pptr, &entry->source); - towire_node_id(pptr, &entry->destination); + towire_node_id(pptr, &entry->node[0]); + towire_node_id(pptr, &entry->node[1]); towire_amount_sat(pptr, entry->sat); - towire_u8(pptr, entry->message_flags); - towire_u8(pptr, entry->channel_flags); + towire_short_channel_id(pptr, &entry->short_channel_id); towire_bool(pptr, entry->public); towire_bool(pptr, entry->local_disabled); - towire_u32(pptr, entry->last_update_timestamp); - towire_u32(pptr, entry->base_fee_msat); - towire_u32(pptr, entry->fee_per_millionth); - towire_u32(pptr, entry->delay); + if (entry->e[0]) { + towire_bool(pptr, true); + towire_gossip_halfchannel_entry(pptr, entry->e[0]); + } else + towire_bool(pptr, false); + + if (entry->e[1]) { + towire_bool(pptr, true); + towire_gossip_halfchannel_entry(pptr, entry->e[1]); + } else + towire_bool(pptr, false); } struct peer_features * diff --git a/lightningd/gossip_msg.h b/lightningd/gossip_msg.h index 5a62e4995..26ab7c898 100644 --- a/lightningd/gossip_msg.h +++ b/lightningd/gossip_msg.h @@ -20,20 +20,25 @@ struct gossip_getnodes_entry { u8 color[3]; }; -struct gossip_getchannels_entry { - struct node_id source, destination; - struct amount_sat sat; - struct short_channel_id short_channel_id; +struct gossip_halfchannel_entry { u8 message_flags; u8 channel_flags; - bool public; - bool local_disabled; u32 last_update_timestamp; u32 delay; u32 base_fee_msat; u32 fee_per_millionth; }; +struct gossip_getchannels_entry { + struct node_id node[2]; + struct amount_sat sat; + struct short_channel_id short_channel_id; + bool public; + bool local_disabled; + /* NULL if we haven't received an update */ + struct gossip_halfchannel_entry *e[2]; +}; + struct gossip_getnodes_entry * fromwire_gossip_getnodes_entry(const tal_t *ctx, const u8 **pptr, size_t *max); void towire_gossip_getnodes_entry(u8 **pptr, @@ -49,8 +54,9 @@ void towire_route_hop(u8 **pprt, const struct route_hop *entry); void fromwire_route_info(const u8 **pprt, size_t *max, struct route_info *entry); void towire_route_info(u8 **pprt, const struct route_info *entry); -void fromwire_gossip_getchannels_entry(const u8 **pptr, size_t *max, - struct gossip_getchannels_entry *entry); +struct gossip_getchannels_entry * +fromwire_gossip_getchannels_entry(const tal_t *ctx, + const u8 **pptr, size_t *max); void towire_gossip_getchannels_entry( u8 **pptr, const struct gossip_getchannels_entry *entry); diff --git a/tools/generate-wire.py b/tools/generate-wire.py index 688cb974e..5dde18660 100755 --- a/tools/generate-wire.py +++ b/tools/generate-wire.py @@ -37,6 +37,7 @@ type2size = { varlen_structs = [ 'peer_features', 'gossip_getnodes_entry', + 'gossip_getchannels_entry', 'failed_htlc', 'utxo', 'bitcoin_tx', diff --git a/wire/wire_io.h b/wire/wire_io.h index 2cdb537ee..fd6d773c3 100644 --- a/wire/wire_io.h +++ b/wire/wire_io.h @@ -5,9 +5,8 @@ #include #include -/* We don't allow > 64M msgs: enough for 483 64k failure msgs. */ -/* FIXME: Too big, but allows the million-channels project at 327077670 bytes */ -#define WIRE_LEN_LIMIT (1 << 29) +/* We don't allow > 128M msgs: enough for more than 1M channels in gossip_getchannels_entry. */ +#define WIRE_LEN_LIMIT (1 << 27) typedef be32 wire_len_t; #define wirelen_to_cpu be32_to_cpu