From 7caa37f0f18c2e9aa3ca69bb3f358ca1835c882c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 17 Apr 2019 17:05:33 +0930 Subject: [PATCH] gossipd: implement Dijkstra. Use a uintmap as our minheap. Note that Dijkstra can give overlength routes, so some checks are disabled. Comparison using gossipd/test/run-bench-find_route 100000 10: Before: 10 (10 succeeded) routes in 100000 nodes in 120087 msec (12008708402 nanoseconds per route) After: 10 (10 succeeded) routes in 100000 nodes in 2269 msec (226925462 nanoseconds per route) Signed-off-by: Rusty Russell --- gossipd/routing.c | 398 +++++++++++++++++++++++++++- gossipd/routing.h | 10 + gossipd/test/run-bench-find_route.c | 29 +- gossipd/test/run-overlong.c | 19 +- 4 files changed, 439 insertions(+), 17 deletions(-) diff --git a/gossipd/routing.c b/gossipd/routing.c index e44f7a94a..265420b41 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -22,6 +22,8 @@ #define SUPERVERBOSE(...) #endif +bool only_dijkstra = false; + /* 365.25 * 24 * 60 / 10 */ #define BLOCKS_PER_YEAR 52596 @@ -424,6 +426,12 @@ struct chan *new_chan(struct routing_state *rstate, /* Too big to reach, but don't overflow if added. */ #define INFINITE AMOUNT_MSAT(0x3FFFFFFFFFFFFFFFULL) +/* We hack a multimap into a uintmap to implement a minheap by cost. + * This is relatively inefficient, containing an array for each cost + * value, assuming there aren't too many at same cost. FIXME: + * implement an array heap */ +typedef UINTMAP(struct node **) unvisited_t; + static void clear_bfg(struct node_map *nodes) { struct node *n; @@ -589,15 +597,363 @@ static bool hc_is_routable(struct routing_state *rstate, && !is_chan_local_disabled(rstate, chan); } +static bool is_unvisited(const struct node *node, + const unvisited_t *unvisited) +{ + struct node **arr; + struct amount_msat cost; + + /* If it's infinite, definitely unvisited */ + if (amount_msat_eq(node->dijkstra.total, INFINITE)) + return true; + + /* Shouldn't happen! */ + if (!amount_msat_add(&cost, node->dijkstra.total, node->dijkstra.risk)) { + status_broken("Can't add cost of node %s + %s", + type_to_string(tmpctx, struct amount_msat, + &node->dijkstra.total), + type_to_string(tmpctx, struct amount_msat, + &node->dijkstra.risk)); + return false; + } + + arr = uintmap_get(unvisited, cost.millisatoshis); + for (size_t i = 0; i < tal_count(arr); i++) { + if (arr[i] == node) + return true; + } + return false; +} + +static void unvisited_add(unvisited_t *unvisited, struct amount_msat cost, + struct node **arr) +{ + uintmap_add(unvisited, cost.millisatoshis, arr); /* Raw: uintmap */ +} + +static struct node **unvisited_del(unvisited_t *unvisited, + struct amount_msat cost) +{ + return uintmap_del(unvisited, cost.millisatoshis); /* Raw: uintmap */ +} + +static void unvisited_del_node(unvisited_t *unvisited, + struct amount_msat cost, + const struct node *node) +{ + size_t i; + struct node **arr; + + /* Remove may reallocate, so we delete and re-add. */ + arr = unvisited_del(unvisited, cost); + for (i = 0; arr[i] != node; i++) + assert(i < tal_count(arr)); + + tal_arr_remove(&arr, i); + if (tal_count(arr) == 0) + tal_free(arr); + else + unvisited_add(unvisited, cost, arr); +} + +static void adjust_unvisited(struct node *node, + unvisited_t *unvisited, + struct amount_msat cost_before, + struct amount_msat total, + struct amount_msat risk, + struct amount_msat cost_after) +{ + struct node **arr; + + /* If it was in unvisited map, remove it. */ + if (!amount_msat_eq(node->dijkstra.total, INFINITE)) + unvisited_del_node(unvisited, cost_before, node); + + /* Update node */ + node->dijkstra.total = total; + node->dijkstra.risk = risk; + + /* Update map of unvisited nodes */ + arr = unvisited_del(unvisited, cost_after); + if (!arr) { + arr = tal_arr(unvisited, struct node *, 1); + arr[0] = node; + } else + tal_arr_expand(&arr, node); + + unvisited_add(unvisited, cost_after, arr); +} + +static void remove_unvisited(struct node *node, unvisited_t *unvisited) +{ + struct amount_msat cost; + + /* Shouldn't happen! */ + if (!amount_msat_add(&cost, node->dijkstra.total, node->dijkstra.risk)) { + status_broken("Can't add unvisited cost of node %s + %s", + type_to_string(tmpctx, struct amount_msat, + &node->dijkstra.total), + type_to_string(tmpctx, struct amount_msat, + &node->dijkstra.risk)); + return; + } + + unvisited_del_node(unvisited, cost, node); +} + +static void update_unvisited_neighbors(struct routing_state *rstate, + struct node *cur, + double riskfactor, + unvisited_t *unvisited) +{ + struct chan_map_iter i; + struct chan *chan; + + /* Consider all neighbors */ + for (chan = first_chan(cur, &i); chan; chan = next_chan(cur, &i)) { + struct amount_msat total, risk, cost_before, cost_after; + int idx = half_chan_to(cur, chan); + struct node *peer = chan->nodes[idx]; + + if (!hc_is_routable(rstate, chan, idx)) + continue; + + if (!is_unvisited(peer, unvisited)) + continue; + + if (!can_reach(&chan->half[idx], + cur->dijkstra.total, cur->dijkstra.risk, + riskfactor, 1.0 /*FIXME*/, &total, &risk)) + continue; + + /* This effectively adds it to the map if it was infinite */ + if (costs_less(total, risk, &cost_after, + peer->dijkstra.total, peer->dijkstra.risk, + &cost_before)) { + SUPERVERBOSE("...%s can reach %s" + " total %s risk %s", + type_to_string(tmpctx, struct node_id, + &cur->id), + type_to_string(tmpctx, struct node_id, + &peer->id), + type_to_string(tmpctx, struct amount_msat, + &total), + type_to_string(tmpctx, struct amount_msat, + &risk)); + adjust_unvisited(peer, unvisited, + cost_before, total, risk, cost_after); + } + } +} + +static struct node *first_unvisited(unvisited_t *unvisited) +{ + u64 idx; + struct node **arr = uintmap_first(unvisited, &idx); + + if (arr) + return arr[0]; + return NULL; +} + +static void dijkstra(struct routing_state *rstate, + const struct node *dst, + double riskfactor, + unvisited_t *unvisited) +{ + struct node *cur; + + while ((cur = first_unvisited(unvisited)) != NULL) { + update_unvisited_neighbors(rstate, cur, riskfactor, unvisited); + remove_unvisited(cur, unvisited); + if (cur == dst) + return; + } +} + +/* Note that we calculated route *backwards*, for fees. So "from" + * here has a high cost, "to" has a cost of exact amount sent. */ +static struct chan **build_route(const tal_t *ctx, + struct routing_state *rstate, + const struct node *from, + const struct node *to, + double riskfactor, + struct amount_msat *fee) +{ + const struct node *i; + struct chan **route, *chan; + + SUPERVERBOSE("Building route from %s (%s) -> %s (%s)", + type_to_string(tmpctx, struct node_id, &from->id), + type_to_string(tmpctx, struct amount_msat, + &from->dijkstra.total), + type_to_string(tmpctx, struct node_id, &to->id), + type_to_string(tmpctx, struct amount_msat, + &to->dijkstra.total)); + /* Never reached? */ + if (amount_msat_eq(from->dijkstra.total, INFINITE)) + return NULL; + + /* Walk to find which neighbors we used */ + route = tal_arr(ctx, struct chan *, 0); + for (i = from; i != to; i = other_node(i, chan)) { + struct chan_map_iter it; + + /* Consider all neighbors */ + for (chan = first_chan(i, &it); chan; chan = next_chan(i, &it)) { + struct node *peer = other_node(i, chan); + struct half_chan *hc = half_chan_from(i, chan); + struct amount_msat total, risk; + + SUPERVERBOSE("CONSIDER: %s -> %s (%s)", + type_to_string(tmpctx, struct node_id, + &i->id), + type_to_string(tmpctx, struct node_id, + &peer->id), + type_to_string(tmpctx, struct amount_msat, + &peer->dijkstra.total)); + + /* If traversing this wasn't possible, ignore */ + if (!hc_is_routable(rstate, chan, !half_chan_to(i, chan))) + continue; + + if (!can_reach(hc, + peer->dijkstra.total, peer->dijkstra.risk, + riskfactor, 1.0 /*FIXME*/, &total, &risk)) + continue; + + /* If this was the path we took, we're done (if there are + * two identical ones, it doesn't matter which) */ + if (amount_msat_eq(total, i->dijkstra.total) + && amount_msat_eq(risk, i->dijkstra.risk)) + break; + } + + if (!chan) { + status_broken("Could not find hop to %s", + type_to_string(tmpctx, struct node_id, + &i->id)); + return tal_free(route); + } + tal_arr_expand(&route, chan); + } + + /* We don't charge ourselves fees, so skip first hop */ + if (!amount_msat_sub(fee, + other_node(from, route[0])->dijkstra.total, + to->dijkstra.total)) { + status_broken("Could not subtract %s - %s for fee", + type_to_string(tmpctx, struct amount_msat, + &other_node(from, route[0]) + ->dijkstra.total), + type_to_string(tmpctx, struct amount_msat, + &to->dijkstra.total)); + return tal_free(route); + } + + return route; +} + +static unvisited_t *dijkstra_prepare(const tal_t *ctx, + struct routing_state *rstate, + struct node *src, + struct amount_msat msat) +{ + struct node_map_iter it; + unvisited_t *unvisited; + struct node *n; + struct node **arr; + + unvisited = tal(ctx, unvisited_t); + uintmap_init(unvisited); + + /* Reset all the information. */ + for (n = node_map_first(rstate->nodes, &it); + n; + n = node_map_next(rstate->nodes, &it)) { + if (n == src) + continue; + n->dijkstra.total = INFINITE; + n->dijkstra.risk = AMOUNT_MSAT(0); + } + + /* Mark start cost: place in unvisited map. */ + src->dijkstra.total = msat; + src->dijkstra.risk = AMOUNT_MSAT(0); + arr = tal_arr(unvisited, struct node *, 1); + arr[0] = src; + unvisited_add(unvisited, msat, arr); + + return unvisited; +} + +static void dijkstra_cleanup(unvisited_t *unvisited) +{ + struct node **arr; + u64 idx; + + /* uintmap uses malloc, so manual cleaning needed */ + while ((arr = uintmap_first(unvisited, &idx)) != NULL) { + tal_free(arr); + uintmap_del(unvisited, idx); + } + tal_free(unvisited); +} + /* riskfactor is already scaled to per-block amount */ static struct chan ** -find_route(const tal_t *ctx, struct routing_state *rstate, - const struct node_id *from, const struct node_id *to, - struct amount_msat msat, - double riskfactor, - double fuzz, const struct siphash_seed *base_seed, - size_t max_hops, - struct amount_msat *fee) +find_route_dijkstra(const tal_t *ctx, struct routing_state *rstate, + const struct node_id *from, const struct node_id *to, + struct amount_msat msat, + double riskfactor, + double fuzz, const struct siphash_seed *base_seed, + size_t max_hops, + struct amount_msat *fee) +{ + struct node *src, *dst; + unvisited_t *unvisited; + + /* Note: we map backwards, since we know the amount of satoshi we want + * at the end, and need to derive how much we need to send. */ + dst = get_node(rstate, from); + src = get_node(rstate, to); + + if (!src) { + status_info("find_route: cannot find %s", + type_to_string(tmpctx, struct node_id, to)); + return NULL; + } else if (!dst) { + status_info("find_route: cannot find myself (%s)", + type_to_string(tmpctx, struct node_id, to)); + return NULL; + } else if (dst == src) { + status_info("find_route: this is %s, refusing to create empty route", + type_to_string(tmpctx, struct node_id, to)); + return NULL; + } + + if (max_hops > ROUTING_MAX_HOPS) { + status_info("find_route: max_hops huge amount %zu > %u", + max_hops, ROUTING_MAX_HOPS); + return NULL; + } + + unvisited = dijkstra_prepare(tmpctx, rstate, src, msat); + dijkstra(rstate, dst, riskfactor, unvisited); + dijkstra_cleanup(unvisited); + + return build_route(ctx, rstate, dst, src, riskfactor, fee); +} + +/* riskfactor is already scaled to per-block amount */ +static struct chan ** +find_route_bfg(const tal_t *ctx, struct routing_state *rstate, + const struct node_id *from, const struct node_id *to, + struct amount_msat msat, + double riskfactor, + double fuzz, const struct siphash_seed *base_seed, + size_t max_hops, + struct amount_msat *fee) { struct chan **route; struct node *n, *src, *dst; @@ -1793,6 +2149,34 @@ u8 *handle_node_announcement(struct routing_state *rstate, const u8 *node_ann) return NULL; } +static struct chan ** +find_route(const tal_t *ctx, struct routing_state *rstate, + const struct node_id *from, const struct node_id *to, + struct amount_msat msat, + double riskfactor, + double fuzz, const struct siphash_seed *base_seed, + size_t max_hops, + struct amount_msat *fee) +{ + struct chan **rd, **rbfg; + + rd = find_route_dijkstra(ctx, rstate, from, to, msat, + riskfactor, + fuzz, base_seed, max_hops, fee); + if (only_dijkstra) + return rd; + + /* Make sure they match */ + rbfg = find_route_bfg(ctx, rstate, from, to, msat, + riskfactor, + fuzz, base_seed, max_hops, fee); + /* FIXME: Dijkstra can give overlength! */ + if (tal_count(rd) < max_hops) + assert(memeq(rd, tal_bytelen(rd), rbfg, tal_bytelen(rbfg))); + tal_free(rd); + return rbfg; +} + struct route_hop *get_route(const tal_t *ctx, struct routing_state *rstate, const struct node_id *source, const struct node_id *destination, diff --git a/gossipd/routing.h b/gossipd/routing.h index 6dd1c3f2a..83572bbfb 100644 --- a/gossipd/routing.h +++ b/gossipd/routing.h @@ -118,6 +118,13 @@ struct node { /* Where that came from. */ struct chan *prev; } bfg[ROUTING_MAX_HOPS+1]; + + struct { + /* Total to get to here from target. */ + struct amount_msat total; + /* Total risk premium of this route. */ + struct amount_msat risk; + } dijkstra; }; const struct node_id *node_map_keyof_node(const struct node *n); @@ -428,4 +435,7 @@ static inline void local_enable_chan(struct routing_state *rstate, /* Helper to convert on-wire addresses format to wireaddrs array */ struct wireaddr *read_addresses(const tal_t *ctx, const u8 *ser); + +/* Temporary to set routing algo */ +extern bool only_dijkstra; #endif /* LIGHTNING_GOSSIPD_ROUTING_H */ diff --git a/gossipd/test/run-bench-find_route.c b/gossipd/test/run-bench-find_route.c index d61bfd3c2..10f8c9956 100644 --- a/gossipd/test/run-bench-find_route.c +++ b/gossipd/test/run-bench-find_route.c @@ -171,12 +171,12 @@ static void populate_random_node(struct routing_state *rstate, u32 randnode = pseudorand(n); add_connection(rstate, nodes, n, randnode, - pseudorand(100), - pseudorand(100), + pseudorand(1000), + pseudorand(1000), pseudorand(144)); add_connection(rstate, nodes, randnode, n, - pseudorand(100), - pseudorand(100), + pseudorand(1000), + pseudorand(1000), pseudorand(144)); } } @@ -205,7 +205,7 @@ int main(int argc, char *argv[]) struct routing_state *rstate; size_t num_nodes = 100, num_runs = 1; struct timemono start, end; - size_t num_success; + size_t route_lengths[ROUTING_MAX_HOPS+1]; struct node_id me; struct node_id *nodes; bool perfme = false; @@ -222,6 +222,7 @@ int main(int argc, char *argv[]) "Run perfme-start and perfme-stop around benchmark"); opt_parse(&argc, argv, opt_log_stderr_exit); + only_dijkstra = true; if (argc > 1) num_nodes = atoi(argv[1]); @@ -230,10 +231,12 @@ int main(int argc, char *argv[]) if (argc > 3) opt_usage_and_exit("[num_nodes [num_runs]]"); + printf("Creating nodes...\n"); nodes = tal_arr(rstate, struct node_id, num_nodes); for (size_t i = 0; i < num_nodes; i++) nodes[i] = nodeid(i); + printf("Populating nodes...\n"); memset(&base_seed, 0, sizeof(base_seed)); for (size_t i = 0; i < num_nodes; i++) populate_random_node(rstate, nodes, i); @@ -241,13 +244,15 @@ int main(int argc, char *argv[]) if (perfme) run("perfme-start"); + printf("Starting...\n"); + memset(route_lengths, 0, sizeof(route_lengths)); start = time_mono(); - num_success = 0; for (size_t i = 0; i < num_runs; i++) { const struct node_id *from = &nodes[pseudorand(num_nodes)]; const struct node_id *to = &nodes[pseudorand(num_nodes)]; struct amount_msat fee; struct chan **route; + size_t num_hops; route = find_route(tmpctx, rstate, from, to, (struct amount_msat){pseudorand(100000)}, @@ -255,7 +260,10 @@ int main(int argc, char *argv[]) 0.75, &base_seed, ROUTING_MAX_HOPS, &fee); - num_success += (route != NULL); + num_hops = tal_count(route); + /* FIXME: Dijkstra can give overlength! */ + if (num_hops < ARRAY_SIZE(route_lengths)) + route_lengths[num_hops]++; tal_free(route); } end = time_mono(); @@ -263,10 +271,13 @@ int main(int argc, char *argv[]) if (perfme) run("perfme-stop"); - printf("%zu (%zu succeeded) routes in %zu nodes in %"PRIu64" msec (%"PRIu64" nanoseconds per route)", - num_runs, num_success, num_nodes, + printf("%zu (%zu succeeded) routes in %zu nodes in %"PRIu64" msec (%"PRIu64" nanoseconds per route)\n", + num_runs, num_runs - route_lengths[0], num_nodes, time_to_msec(timemono_between(end, start)), time_to_nsec(time_divide(timemono_between(end, start), num_runs))); + for (size_t i = 0; i < ARRAY_SIZE(route_lengths); i++) + if (route_lengths[i]) + printf(" Length %zu: %zu\n", i, route_lengths[i]); tal_free(tmpctx); secp256k1_context_destroy(secp256k1_ctx); diff --git a/gossipd/test/run-overlong.c b/gossipd/test/run-overlong.c index 91eee4aa0..bc90dc320 100644 --- a/gossipd/test/run-overlong.c +++ b/gossipd/test/run-overlong.c @@ -108,7 +108,7 @@ static void node_id_from_privkey(const struct privkey *p, struct node_id *id) node_id_from_pubkey(id, &k); } -#define NUM_NODES 21 +#define NUM_NODES (ROUTING_MAX_HOPS + 1) /* We create an arrangement of nodes, each node N connected to N+1 and * to node 1. The cost for each N to N+1 route is 1, for N to 1 is @@ -156,6 +156,10 @@ int main(void) hc->channel_flags = node_id_idx(&ids[i-1], &ids[i]); hc->htlc_minimum = AMOUNT_MSAT(0); hc->htlc_maximum = AMOUNT_MSAT(1000000 * 1000); + SUPERVERBOSE("Joining %s to %s, fee %u", + type_to_string(tmpctx, struct node_id, &ids[i-1]), + type_to_string(tmpctx, struct node_id, &ids[i]), + (int)hc->base_fee); if (i <= 2) continue; @@ -171,14 +175,27 @@ int main(void) hc->channel_flags = node_id_idx(&ids[1], &ids[i]); hc->htlc_minimum = AMOUNT_MSAT(0); hc->htlc_maximum = AMOUNT_MSAT(1000000 * 1000); + SUPERVERBOSE("Joining %s to %s, fee %u", + type_to_string(tmpctx, struct node_id, &ids[1]), + type_to_string(tmpctx, struct node_id, &ids[i]), + (int)hc->base_fee); } for (size_t i = ROUTING_MAX_HOPS; i > 1; i--) { struct amount_msat fee; + SUPERVERBOSE("%s -> %s:", + type_to_string(tmpctx, struct node_id, &ids[0]), + type_to_string(tmpctx, struct node_id, &ids[NUM_NODES-1])); + route = find_route(tmpctx, rstate, &ids[0], &ids[NUM_NODES-1], AMOUNT_MSAT(1000), 0, 0.0, NULL, i, &fee); assert(route); + /* FIXME: dijkstra ignores maximum length requirement! */ + if (only_dijkstra) { + assert(tal_count(route) == NUM_NODES-1); + continue; + } assert(tal_count(route) == i); if (i != ROUTING_MAX_HOPS) assert(amount_msat_greater(fee, last_fee));