seeker: add code to check range of scids.

Once we've finished streaming gossip from the first peer, we ask a
random peer (maybe the same one) for all short_channel_ids in the last
6 blocks from the latest channel we know about.

If this reveals new channels we didn't know about, we expand the probe
by a factor of 2 each time.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

gossipd/seeker.c

@@ -21,6 +21,12 @@ enum seeker_state {
 	/* Still streaming gossip from single peer. */
 	STARTING_UP_FIRSTPEER,
 
+	/* Probing scids: need peer to check startup really finished. */
+	PROBING_SCIDS_NEED_PEER,
+
+	/* Probing: checking our startup really is finished. */
+	PROBING_SCIDS,
+
 	/* Normal running. */
 	NORMAL,
 };
@@ -37,6 +43,9 @@ struct seeker {
 	/* Channels we've heard about, but don't know. */
 	struct short_channel_id *unknown_scids;
 
+	/* Range of scid blocks we've probed. */
+	size_t scid_probe_start, scid_probe_end;
+
 	/* Timestamp of gossip store (or 0). */
 	u32 last_gossip_timestamp;
 
@@ -153,8 +162,118 @@ static bool peer_has_gossip_queries(const struct peer *peer)
 	return peer->gossip_queries_feature;
 }
 
+static bool peer_can_take_range_query(const struct peer *peer)
+{
+	return peer->gossip_queries_feature
+		&& !peer->query_channel_blocks;
+}
+
+static void seek_any_unknown_scids(struct seeker *seeker)
+{
+	/* FIXME: implement! */
+}
+
+/* Returns true and sets first_blocknum and number_of_blocks if
+ * there's more to find. */
+static bool next_block_range(struct seeker *seeker,
+			     u32 prev_num_blocks,
+			     u32 *first_blocknum, u32 *number_of_blocks)
+{
+	const u32 current_height = seeker->daemon->current_blockheight;
+
+	/* We always try to get twice as many as last time. */
+	*number_of_blocks = prev_num_blocks * 2;
+
+	if (seeker->scid_probe_start > 0) {
+		/* Enlarge probe to cover prior blocks, but twice as many. */
+		if (*number_of_blocks > seeker->scid_probe_start) {
+			*number_of_blocks = seeker->scid_probe_start;
+			*first_blocknum = 0;
+		} else {
+			*first_blocknum
+				= seeker->scid_probe_start - *number_of_blocks;
+		}
+		seeker->scid_probe_start = *first_blocknum;
+		return true;
+	}
+
+	/* We allow 6 new blocks since we started; they should be empty anyway */
+	if (seeker->scid_probe_end + 6 < current_height) {
+		if (seeker->scid_probe_end + *number_of_blocks > current_height)
+			*number_of_blocks
+				= current_height - seeker->scid_probe_end;
+		*first_blocknum = seeker->scid_probe_end + 1;
+		seeker->scid_probe_end = *first_blocknum + *number_of_blocks - 1;
+		return true;
+	}
+
+	/* No more to find. */
+	return false;
+}
+
+static void process_scid_probe(struct peer *peer,
+			       u32 first_blocknum, u32 number_of_blocks,
+			       const struct short_channel_id *scids,
+			       bool complete)
+{
+	struct seeker *seeker = peer->daemon->seeker;
+	bool new_unknown_scids = false;
+
+	assert(seeker->random_peer_softref == peer);
+	clear_softref(seeker, &seeker->random_peer_softref);
+
+	for (size_t i = 0; i < tal_count(scids); i++) {
+		struct chan *c = get_channel(seeker->daemon->rstate, &scids[i]);
+		if (c)
+			continue;
+
+		new_unknown_scids |= add_unknown_scid(seeker, &scids[i]);
+	}
+
+	/* No new unknown scids, or no more to ask?  We give some wiggle
+	 * room in case blocks came in since we started. */
+	if (new_unknown_scids
+	    && next_block_range(seeker, number_of_blocks,
+				&first_blocknum, &number_of_blocks)) {
+		/* This must return a peer, since we have the current peer! */
+		peer = random_peer(seeker->daemon, peer_can_take_range_query);
+		assert(peer);
+		selected_peer(seeker, peer);
+
+		query_channel_range(seeker->daemon, peer,
+				    first_blocknum, number_of_blocks,
+				    process_scid_probe);
+		return;
+	}
+
+	/* Probe finished. */
+	seeker->state = NORMAL;
+	seek_any_unknown_scids(seeker);
+	return;
+}
+
+/* Pick a peer, ask it for a few scids, to check. */
+static void peer_gossip_probe_scids(struct seeker *seeker)
+{
+	struct peer *peer;
+
+	peer = random_peer(seeker->daemon, peer_can_take_range_query);
+	if (!peer)
+		return;
+	selected_peer(seeker, peer);
+
+	/* This calls process_scid_probe when we get the reply. */
+	query_channel_range(seeker->daemon, peer,
+			    seeker->scid_probe_start,
+			    seeker->scid_probe_end - seeker->scid_probe_start + 1,
+			    process_scid_probe);
+	seeker->state = PROBING_SCIDS;
+}
+
 static void check_firstpeer(struct seeker *seeker)
 {
+	struct chan *c;
+	u64 index;
 	struct peer *peer = seeker->random_peer_softref, *p;
 
 	/* It might have died, pick another. */
@@ -177,16 +296,40 @@ static void check_firstpeer(struct seeker *seeker)
 	if (peer_made_progress(seeker))
 		return;
 
-	/* Begin normal gossip regime */
+	/* Other peers can gossip now. */
 	status_debug("seeker: startup peer finished");
 	clear_softref(seeker, &seeker->random_peer_softref);
-	seeker->state = NORMAL;
 	list_for_each(&seeker->daemon->peers, p, list) {
 		if (p == peer)
 			continue;
 
 		normal_gossip_start(seeker, p);
 	}
+
+	/* We always look up 6 prior to last we have */
+	c = uintmap_last(&seeker->daemon->rstate->chanmap, &index);
+	if (c && short_channel_id_blocknum(&c->scid) > 6) {
+		seeker->scid_probe_start = short_channel_id_blocknum(&c->scid) - 6;
+	} else {
+		seeker->scid_probe_start = 0;
+	}
+	seeker->scid_probe_end = seeker->daemon->current_blockheight;
+	seeker->state = PROBING_SCIDS_NEED_PEER;
+	peer_gossip_probe_scids(seeker);
+}
+
+static void check_scid_probing(struct seeker *seeker)
+{
+	/* FIXME: Time them out of they don't respond to gossip */
+	struct peer *peer = seeker->random_peer_softref;
+
+	/* It might have died, pick another. */
+	if (!peer) {
+		status_debug("seeker: scid probing peer died, re-choosing");
+		seeker->state = PROBING_SCIDS_NEED_PEER;
+		peer_gossip_probe_scids(seeker);
+		return;
+	}
 }
 
 /* Periodic timer to see how our gossip is going. */
@@ -198,8 +341,14 @@ static void seeker_check(struct seeker *seeker)
 	case STARTING_UP_FIRSTPEER:
 		check_firstpeer(seeker);
 		break;
+	case PROBING_SCIDS_NEED_PEER:
+		peer_gossip_probe_scids(seeker);
+		break;
+	case PROBING_SCIDS:
+		check_scid_probing(seeker);
+		break;
 	case NORMAL:
-		/* FIXME: Check! */
+		seek_any_unknown_scids(seeker);
 		break;
 	}
 
@@ -221,6 +370,12 @@ void seeker_setup_peer_gossip(struct seeker *seeker, struct peer *peer)
 	case STARTING_UP_FIRSTPEER:
 		/* Waiting for seeker_check to release us */
 		return;
+
+	/* In these states, we set up peers to stream gossip normally */
+	case PROBING_SCIDS_NEED_PEER:
+		peer_gossip_probe_scids(seeker);
+		/* fall thru */
+	case PROBING_SCIDS:
 	case NORMAL:
 		normal_gossip_start(seeker, peer);
 		return;

gossipd/test/run-extended-info.c

@@ -47,9 +47,6 @@ const u8 *gossip_store_get(const tal_t *ctx UNNEEDED,
 /* Generated stub for master_badmsg */
 void master_badmsg(u32 type_expected UNNEEDED, const u8 *msg)
 { fprintf(stderr, "master_badmsg called!\n"); abort(); }
-/* Generated stub for process_scids */
-void process_scids(struct daemon *daemon UNNEEDED, const struct short_channel_id *scids UNNEEDED)
-{ fprintf(stderr, "process_scids called!\n"); abort(); }
 /* Generated stub for queue_peer_from_store */
 void queue_peer_from_store(struct peer *peer UNNEEDED,
 			   const struct broadcastable *bcast UNNEEDED)

gossipd/test/run-next_block_range.c

@@ -0,0 +1,113 @@
+#include "../seeker.c"
+#include <ccan/err/err.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+/* AUTOGENERATED MOCKS START */
+/* Generated stub for new_reltimer_ */
+struct oneshot *new_reltimer_(struct timers *timers UNNEEDED,
+			      const tal_t *ctx UNNEEDED,
+			      struct timerel expire UNNEEDED,
+			      void (*cb)(void *) UNNEEDED, void *arg UNNEEDED)
+{ fprintf(stderr, "new_reltimer_ called!\n"); abort(); }
+/* Generated stub for query_channel_range */
+bool query_channel_range(struct daemon *daemon UNNEEDED,
+			 struct peer *peer UNNEEDED,
+			 u32 first_blocknum UNNEEDED, u32 number_of_blocks UNNEEDED,
+			 void (*cb)(struct peer *peer UNNEEDED,
+				    u32 first_blocknum UNNEEDED, u32 number_of_blocks UNNEEDED,
+				    const struct short_channel_id *scids UNNEEDED,
+				    bool complete))
+{ fprintf(stderr, "query_channel_range called!\n"); abort(); }
+/* Generated stub for query_short_channel_ids */
+bool query_short_channel_ids(struct daemon *daemon UNNEEDED,
+			     struct peer *peer UNNEEDED,
+			     const struct short_channel_id *scids UNNEEDED,
+			     void (*cb)(struct peer *peer UNNEEDED, bool complete))
+{ fprintf(stderr, "query_short_channel_ids called!\n"); abort(); }
+/* Generated stub for queue_peer_msg */
+void queue_peer_msg(struct peer *peer UNNEEDED, const u8 *msg TAKES UNNEEDED)
+{ fprintf(stderr, "queue_peer_msg called!\n"); abort(); }
+/* Generated stub for random_peer */
+struct peer *random_peer(struct daemon *daemon UNNEEDED,
+			 bool (*check_peer)(const struct peer *peer))
+{ fprintf(stderr, "random_peer called!\n"); abort(); }
+/* Generated stub for status_fmt */
+void status_fmt(enum log_level level UNNEEDED, const char *fmt UNNEEDED, ...)
+
+{ fprintf(stderr, "status_fmt called!\n"); abort(); }
+/* AUTOGENERATED MOCKS END */
+
+static void test_block_range(struct seeker *seeker,
+			     u32 blockheight,
+			     u32 first, u32 last,
+			     ...)
+{
+	int start, end, num;
+	u32 first_blocknum, number_of_blocks;
+	va_list ap;
+
+	seeker->daemon->current_blockheight = blockheight;
+	seeker->scid_probe_start = first;
+	seeker->scid_probe_end = last;
+
+	num = last - first + 1;
+	va_start(ap, last);
+	while ((start = va_arg(ap, int)) != -1) {
+		end = va_arg(ap, int);
+		if (!next_block_range(seeker, num,
+				      &first_blocknum, &number_of_blocks))
+			abort();
+		if (first_blocknum != start
+		    || number_of_blocks != end - start + 1) {
+			errx(1, "Expected %u-%u but got %u-%u",
+			     start, end,
+			     first_blocknum, first_blocknum+number_of_blocks-1);
+		}
+		num = end - start + 1;
+	}
+	if (next_block_range(seeker, num, &first_blocknum, &number_of_blocks))
+		abort();
+	va_end(ap);
+}
+
+int main(void)
+{
+	struct seeker *seeker = tal(NULL, struct seeker);
+
+	setup_locale();
+
+	seeker->daemon = tal(seeker, struct daemon);
+
+	/* Case where we start at beginning */
+	test_block_range(seeker, 100,
+			 0, 0,
+			 1, 2,
+			 3, 6,
+			 7, 14,
+			 15, 30,
+			 31, 62,
+			 63, 100,
+			 -1);
+
+	/* Case where we start at end */
+	test_block_range(seeker, 100,
+			 100, 100,
+			 98, 99,
+			 94, 97,
+			 86, 93,
+			 70, 85,
+			 38, 69,
+			 0, 37,
+			 -1);
+
+	/* Start in the middle. */
+	test_block_range(seeker, 100,
+			 50, 59,
+			 30, 49,
+			 0, 29,
+			 60, 100,
+			 -1);
+
+	tal_free(seeker);
+}