watchtower: replace taskpipeline with disk overflow queue

watchtower/wtclient/client.go

@@ -244,6 +244,10 @@ type Config struct {
 	// number of blocks to delay closing a session after its last channel
 	// has been closed.
 	SessionCloseRange uint32
+
+	// MaxTasksInMemQueue is the maximum number of backup tasks that should
+	// be kept in-memory. Any more tasks will overflow to disk.
+	MaxTasksInMemQueue uint64
 }
 
 // BreachRetributionBuilder is a function that can be used to construct a
@@ -297,7 +301,7 @@ type TowerClient struct {
 
 	log btclog.Logger
 
-	pipeline *taskPipeline
+	pipeline *DiskOverflowQueue[*wtdb.BackupID]
 
 	negotiator        SessionNegotiator
 	candidateTowers   TowerCandidateIterator
@@ -359,10 +363,21 @@ func New(config *Config) (*TowerClient, error) {
 		return nil, err
 	}
 
+	var (
+		policy  = cfg.Policy.BlobType.String()
+		queueDB = cfg.DB.GetDBQueue([]byte(policy))
+	)
+	queue, err := NewDiskOverflowQueue[*wtdb.BackupID](
+		queueDB, cfg.MaxTasksInMemQueue, plog,
+	)
+	if err != nil {
+		return nil, err
+	}
+
 	c := &TowerClient{
 		cfg:                  cfg,
 		log:                  plog,
-		pipeline:             newTaskPipeline(plog),
+		pipeline:             queue,
 		chanCommitHeights:    make(map[lnwire.ChannelID]uint64),
 		activeSessions:       make(sessionQueueSet),
 		summaries:            chanSummaries,
@@ -675,6 +690,7 @@ func (c *TowerClient) Start() error {
 
 // Stop idempotently initiates a graceful shutdown of the watchtower client.
 func (c *TowerClient) Stop() error {
+	var returnErr error
 	c.stopped.Do(func() {
 		c.log.Debugf("Stopping watchtower client")
 
@@ -697,7 +713,10 @@ func (c *TowerClient) Stop() error {
 		// updates from being accepted. In practice, the links should be
 		// shutdown before the client has been stopped, so all updates
 		// would have been added prior.
-		c.pipeline.Stop()
+		err := c.pipeline.Stop()
+		if err != nil {
+			returnErr = err
+		}
 
 		// 3. Once the backup queue has shutdown, wait for the main
 		// dispatcher to exit. The backup queue will signal it's
@@ -728,7 +747,8 @@ func (c *TowerClient) Stop() error {
 
 		c.log.Debugf("Client successfully stopped, stats: %s", c.stats)
 	})
-	return nil
+
+	return returnErr
 }
 
 // ForceQuit idempotently initiates an unclean shutdown of the watchtower
@@ -741,7 +761,10 @@ func (c *TowerClient) ForceQuit() {
 		// updates from being accepted. In practice, the links should be
 		// shutdown before the client has been stopped, so all updates
 		// would have been added prior.
-		c.pipeline.ForceQuit()
+		err := c.pipeline.Stop()
+		if err != nil {
+			c.log.Errorf("could not stop backup queue: %v", err)
+		}
 
 		// 2. Once the backup queue has shutdown, wait for the main
 		// dispatcher to exit. The backup queue will signal it's
@@ -840,7 +863,7 @@ func (c *TowerClient) BackupState(chanID *lnwire.ChannelID,
 		CommitHeight: stateNum,
 	}
 
-	return c.pipeline.QueueBackupTask(id)
+	return c.pipeline.QueueBackupID(id)
 }
 
 // nextSessionQueue attempts to fetch an active session from our set of
@@ -1327,7 +1350,7 @@ func (c *TowerClient) backupDispatcher() {
 
 			// Process each backup task serially from the queue of
 			// revoked states.
-			case task, ok := <-c.pipeline.NewBackupTasks():
+			case task, ok := <-c.pipeline.NextBackupID():
 				// All backups in the pipeline have been
 				// processed, it is now safe to exit.
 				if !ok {
@@ -1639,8 +1662,6 @@ func (c *TowerClient) AddTower(addr *lnwire.NetAddress) error {
 	}:
 	case <-c.pipeline.quit:
 		return ErrClientExiting
-	case <-c.pipeline.forceQuit:
-		return ErrClientExiting
 	}
 
 	select {
@@ -1648,8 +1669,6 @@ func (c *TowerClient) AddTower(addr *lnwire.NetAddress) error {
 		return err
 	case <-c.pipeline.quit:
 		return ErrClientExiting
-	case <-c.pipeline.forceQuit:
-		return ErrClientExiting
 	}
 }
 
@@ -1706,8 +1725,6 @@ func (c *TowerClient) RemoveTower(pubKey *btcec.PublicKey,
 	}:
 	case <-c.pipeline.quit:
 		return ErrClientExiting
-	case <-c.pipeline.forceQuit:
-		return ErrClientExiting
 	}
 
 	select {
@@ -1715,8 +1732,6 @@ func (c *TowerClient) RemoveTower(pubKey *btcec.PublicKey,
 		return err
 	case <-c.pipeline.quit:
 		return ErrClientExiting
-	case <-c.pipeline.forceQuit:
-		return ErrClientExiting
 	}
 }