channeldb+discovery: use timestamps to maybe revive zombie

discovery/syncer.go

@@ -180,6 +180,9 @@ const (
 	// requestBatchSize is the maximum number of channels we will query the
 	// remote peer for in a QueryShortChanIDs message.
 	requestBatchSize = 500
+
+	// filterSemaSize is the capacity of gossipFilterSema.
+	filterSemaSize = 5
 )
 
 var (
@@ -372,7 +375,7 @@ type GossipSyncer struct {
 
 	// bufferedChanRangeReplies is used in the waitingQueryChanReply to
 	// buffer all the chunked response to our query.
-	bufferedChanRangeReplies []lnwire.ShortChannelID
+	bufferedChanRangeReplies []channeldb.ChannelUpdateInfo
 
 	// numChanRangeRepliesRcvd is used to track the number of replies
 	// received as part of a QueryChannelRange. This field is primarily used
@@ -398,6 +401,8 @@ type GossipSyncer struct {
 
 	sync.Mutex
 
+	gossipFilterSema chan struct{}
+
 	quit chan struct{}
 	wg   sync.WaitGroup
 }
@@ -426,6 +431,11 @@ func newGossipSyncer(cfg gossipSyncerCfg) *GossipSyncer {
 		interval, cfg.maxUndelayedQueryReplies,
 	)
 
+	filterSema := make(chan struct{}, filterSemaSize)
+	for i := 0; i < filterSemaSize; i++ {
+		filterSema <- struct{}{}
+	}
+
 	return &GossipSyncer{
 		cfg:                cfg,
 		rateLimiter:        rateLimiter,
@@ -433,6 +443,7 @@ func newGossipSyncer(cfg gossipSyncerCfg) *GossipSyncer {
 		historicalSyncReqs: make(chan *historicalSyncReq),
 		gossipMsgs:         make(chan lnwire.Message, 100),
 		queryMsgs:          make(chan lnwire.Message, 100),
+		gossipFilterSema:   filterSema,
 		quit:               make(chan struct{}),
 	}
 }
@@ -819,9 +830,31 @@ func (g *GossipSyncer) processChanRangeReply(msg *lnwire.ReplyChannelRange) erro
 	}
 
 	g.prevReplyChannelRange = msg
-	g.bufferedChanRangeReplies = append(
-		g.bufferedChanRangeReplies, msg.ShortChanIDs...,
-	)
+	if len(msg.Timestamps) != 0 &&
+		len(msg.Timestamps) != len(msg.ShortChanIDs) {
+
+		return fmt.Errorf("number of timestamps not equal to " +
+			"number of SCIDs")
+	}
+
+	for i, scid := range msg.ShortChanIDs {
+		info := channeldb.ChannelUpdateInfo{
+			ShortChannelID: scid,
+		}
+
+		if len(msg.Timestamps) != 0 {
+			t1 := time.Unix(int64(msg.Timestamps[i].Timestamp1), 0)
+			info.Node1UpdateTimestamp = t1
+
+			t2 := time.Unix(int64(msg.Timestamps[i].Timestamp2), 0)
+			info.Node2UpdateTimestamp = t2
+		}
+
+		g.bufferedChanRangeReplies = append(
+			g.bufferedChanRangeReplies, info,
+		)
+	}
+
 	switch g.cfg.encodingType {
 	case lnwire.EncodingSortedPlain:
 		g.numChanRangeRepliesRcvd++
@@ -868,6 +901,7 @@ func (g *GossipSyncer) processChanRangeReply(msg *lnwire.ReplyChannelRange) erro
 	// which channels they know of that we don't.
 	newChans, err := g.cfg.channelSeries.FilterKnownChanIDs(
 		g.cfg.chainHash, g.bufferedChanRangeReplies,
+		g.cfg.isStillZombieChannel,
 	)
 	if err != nil {
 		return fmt.Errorf("unable to filter chan ids: %v", err)
@@ -1107,10 +1141,20 @@ func (g *GossipSyncer) replyChanRangeQuery(query *lnwire.QueryChannelRange) erro
 		lastHeight   uint32
 		channelChunk []channeldb.ChannelUpdateInfo
 	)
+
+	// chunkSize is the maximum number of SCIDs that we can safely put in a
+	// single message. If we also need to include timestamps though, then
+	// this number is halved since encoding two timestamps takes the same
+	// number of bytes as encoding an SCID.
+	chunkSize := g.cfg.chunkSize
+	if withTimestamps {
+		chunkSize /= 2
+	}
+
 	for _, channelRange := range channelRanges {
 		channels := channelRange.Channels
 		numChannels := int32(len(channels))
-		numLeftToAdd := g.cfg.chunkSize - int32(len(channelChunk))
+		numLeftToAdd := chunkSize - int32(len(channelChunk))
 
 		// Include the current block in the ongoing chunk if it can fit
 		// and move on to the next block.
@@ -1126,6 +1170,7 @@ func (g *GossipSyncer) replyChanRangeQuery(query *lnwire.QueryChannelRange) erro
 		// to.
 		log.Infof("GossipSyncer(%x): sending range chunk of size=%v",
 			g.cfg.peerPub[:], len(channelChunk))
+
 		lastHeight = channelRange.Height - 1
 		err := sendReplyForChunk(
 			channelChunk, firstHeight, lastHeight, false,
@@ -1140,15 +1185,15 @@ func (g *GossipSyncer) replyChanRangeQuery(query *lnwire.QueryChannelRange) erro
 		// this isn't an issue since we'll randomly shuffle them and we
 		// assume a historical gossip sync is performed at a later time.
 		firstHeight = channelRange.Height
-		chunkSize := numChannels
-		exceedsChunkSize := numChannels > g.cfg.chunkSize
+		finalChunkSize := numChannels
+		exceedsChunkSize := numChannels > chunkSize
 		if exceedsChunkSize {
 			rand.Shuffle(len(channels), func(i, j int) {
 				channels[i], channels[j] = channels[j], channels[i]
 			})
-			chunkSize = g.cfg.chunkSize
+			finalChunkSize = chunkSize
 		}
-		channelChunk = channels[:chunkSize]
+		channelChunk = channels[:finalChunkSize]
 
 		// Sort the chunk once again if we had to shuffle it.
 		if exceedsChunkSize {
@@ -1164,6 +1209,7 @@ func (g *GossipSyncer) replyChanRangeQuery(query *lnwire.QueryChannelRange) erro
 	// Send the remaining chunk as the final reply.
 	log.Infof("GossipSyncer(%x): sending final chan range chunk, size=%v",
 		g.cfg.peerPub[:], len(channelChunk))
+
 	return sendReplyForChunk(
 		channelChunk, firstHeight, query.LastBlockHeight(), true,
 	)
@@ -1267,10 +1313,19 @@ func (g *GossipSyncer) ApplyGossipFilter(filter *lnwire.GossipTimestampRange) er
 		return nil
 	}
 
+	select {
+	case <-g.gossipFilterSema:
+	case <-g.quit:
+		return ErrGossipSyncerExiting
+	}
+
 	// We'll conclude by launching a goroutine to send out any updates.
 	g.wg.Add(1)
 	go func() {
 		defer g.wg.Done()
+		defer func() {
+			g.gossipFilterSema <- struct{}{}
+		}()
 
 		for _, msg := range newUpdatestoSend {
 			err := g.cfg.sendToPeerSync(msg)