itest+lntest: migrate lntemp to lntest

This commit performs the takeover that `lntemp` is now promoted to be
`lntest`, and the scaffolding is now removed as all the refactoring is
finished!

lntest/README.md

@@ -0,0 +1,95 @@
+# `lntest`
+
+`lntest` is a package which holds the components used for the `lnd`’s
+integration tests. It is responsible for managing `lnd` nodes, chain backends
+and miners, advancing nodes’ states and providing assertions.
+
+### Quick Start
+
+A simple example to run the integration test.
+
+```go
+func TestFoo(t *testing.T) {
+	// Get the binary path and setup the harness test.
+	//
+	// TODO: define the binary path to lnd and the name of the database
+	// backend.
+	harnessTest := lntemp.SetupHarness(t, binary, *dbBackendFlag)
+	defer harnessTest.Stop()
+
+	// Setup standby nodes, Alice and Bob, which will be alive and shared
+	// among all the test cases.
+	harnessTest.SetupStandbyNodes()
+
+	// Run the subset of the test cases selected in this tranche.
+	//
+	// TODO: define your own testCases.
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.Name, func(st *testing.T) {
+			// Create a separate harness test for the testcase to
+			// avoid overwriting the external harness test that is
+			// tied to the parent test.
+			ht := harnessTest.Subtest(st)
+
+			// Run the test cases.
+			ht.RunTestCase(tc)
+		})
+	}
+}
+```
+
+### Package Structure
+
+This package has four major components, `HarnessTest`, `HarnessMiner`,
+`node.HarnessNode` and `rpc.HarnessRPC`, with the following architecture,
+
+```
++----------------------------------------------------------+
+|                                                          |
+|                        HarnessTest                       |
+|                                                          |
+| +----------------+  +----------------+  +--------------+ |
+| |   HarnessNode  |  |   HarnessNode  |  | HarnessMiner | |
+| |                |  |                |  +--------------+ |
+| | +------------+ |  | +------------+ |                   |
+| | | HarnessRPC | |  | | HarnessRPC | |  +--------------+ |
+| | +------------+ |  | +------------+ |  | HarnessMiner | |
+| +----------------+  +----------------+  +--------------+ |
++----------------------------------------------------------+
+```
+
+- `HarnessRPC` holds all the RPC clients and adds a layer over all the RPC
+  methods to assert no error happened at the RPC level.
+
+- `HarnessNode` builds on top of the `HarnessRPC`. It is responsible for
+  managing the `lnd` node, including start and stop pf the `lnd` process,
+  authentication of the gRPC connection, topology subscription(`NodeWatcher`)
+  and maintains an internal state(`NodeState`).
+
+- `HarnessMiner` builds on top of `btcd`’s `rcptest.Harness` and is responsilbe
+  for managing blocks and the mempool.
+
+- `HarnessTest` builds on top of `testing.T` and can be viewed as the assertion
+  machine. It provides multiple ways to initialize a node, such as with/without
+  seed, backups, etc. It also handles interactions between nodes like
+  connecting nodes and opening/closing channels so it’s easier to acquire or
+  validate a desired test states such as node’s balance, mempool condition,
+  etc.
+
+### Standby Nodes
+
+Standby nodes are `HarnessNode`s created when initializing the integration test
+and stay alive across all the test cases. Creating a new node is not without a
+cost. With block height increasing, it takes significantly longer to initialize
+a new node and wait for it to be synced. Standby nodes, however, don’t have
+this problem as they are digesting blocks all the time. Thus it’s encouraged to
+use standby nodes wherever possible.
+
+Currently there are two standby nodes, Alice and Bob. Their internal states are
+recorded and taken into account when `HarnessTest` makes assertions. When
+making a new test case using `Subtest`, there’s a cleanup function which
+further validates the current test case has no dangling uncleaned states, such
+as transactions left in mempool, open channels, etc.
+

lntest/bitcoind_common.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/rpcclient"
-	"github.com/lightningnetwork/lnd/lntemp/node"
+	"github.com/lightningnetwork/lnd/lntest/node"
 )
 
 // logDirPattern is the pattern of the name of the temporary log directory.

lntest/btcd.go

@@ -15,7 +15,7 @@ import (
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/integration/rpctest"
 	"github.com/btcsuite/btcd/rpcclient"
-	"github.com/lightningnetwork/lnd/lntemp/node"
+	"github.com/lightningnetwork/lnd/lntest/node"
 )
 
 // logDirPattern is the pattern of the name of the temporary log directory.

lntest/fee_service.go

@@ -0,0 +1,140 @@
+package lntest
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"sync"
+	"testing"
+
+	"github.com/lightningnetwork/lnd/lntest/node"
+	"github.com/lightningnetwork/lnd/lnwallet/chainfee"
+	"github.com/stretchr/testify/require"
+)
+
+// WebFeeService defines an interface that's used to provide fee estimation
+// service used in the integration tests. It must provide an URL so that a lnd
+// node can be started with the flag `--feeurl` and uses the customized fee
+// estimator.
+type WebFeeService interface {
+	// Start starts the service.
+	Start() error
+
+	// Stop stops the service.
+	Stop() error
+
+	// URL returns the service's endpoint.
+	URL() string
+
+	// SetFeeRate sets the estimated fee rate for a given confirmation
+	// target.
+	SetFeeRate(feeRate chainfee.SatPerKWeight, conf uint32)
+}
+
+const (
+	// feeServiceTarget is the confirmation target for which a fee estimate
+	// is returned. Requests for higher confirmation targets will fall back
+	// to this.
+	feeServiceTarget = 1
+
+	// DefaultFeeRateSatPerKw specifies the default fee rate used in the
+	// tests.
+	DefaultFeeRateSatPerKw = 12500
+)
+
+// FeeService runs a web service that provides fee estimation information.
+type FeeService struct {
+	*testing.T
+
+	feeRateMap map[uint32]uint32
+	url        string
+
+	srv  *http.Server
+	wg   sync.WaitGroup
+	lock sync.Mutex
+}
+
+// Compile-time check for the WebFeeService interface.
+var _ WebFeeService = (*FeeService)(nil)
+
+// Start spins up a go-routine to serve fee estimates.
+func NewFeeService(t *testing.T) *FeeService {
+	port := node.NextAvailablePort()
+	f := FeeService{
+		T: t,
+		url: fmt.Sprintf(
+			"http://localhost:%v/fee-estimates.json", port,
+		),
+	}
+
+	// Initialize default fee estimate.
+	f.feeRateMap = map[uint32]uint32{
+		feeServiceTarget: DefaultFeeRateSatPerKw,
+	}
+
+	listenAddr := fmt.Sprintf(":%v", port)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/fee-estimates.json", f.handleRequest)
+
+	f.srv = &http.Server{
+		Addr:    listenAddr,
+		Handler: mux,
+	}
+	return &f
+}
+
+// Start starts the web server.
+func (f *FeeService) Start() error {
+	f.wg.Add(1)
+	go func() {
+		defer f.wg.Done()
+
+		if err := f.srv.ListenAndServe(); err != http.ErrServerClosed {
+			require.NoErrorf(f, err, "cannot start fee api")
+		}
+	}()
+
+	return nil
+}
+
+// handleRequest handles a client request for fee estimates.
+func (f *FeeService) handleRequest(w http.ResponseWriter, r *http.Request) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	bytes, err := json.Marshal(
+		struct {
+			Fees map[uint32]uint32 `json:"fee_by_block_target"`
+		}{
+			Fees: f.feeRateMap,
+		},
+	)
+	require.NoErrorf(f, err, "cannot serialize estimates")
+
+	_, err = io.WriteString(w, string(bytes))
+	require.NoError(f, err, "cannot send estimates")
+}
+
+// Stop stops the web server.
+func (f *FeeService) Stop() error {
+	err := f.srv.Shutdown(context.Background())
+	require.NoError(f, err, "cannot stop fee api")
+
+	f.wg.Wait()
+	return nil
+}
+
+// SetFeeRate sets a fee for the given confirmation target.
+func (f *FeeService) SetFeeRate(fee chainfee.SatPerKWeight, conf uint32) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	f.feeRateMap[conf] = uint32(fee.FeePerKVByte())
+}
+
+// URL returns the service endpoint.
+func (f *FeeService) URL() string {
+	return f.url
+}

lntest/harness_miner.go

@@ -0,0 +1,438 @@
+package lntest
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcd/btcjson"
+	"github.com/btcsuite/btcd/btcutil"
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/btcsuite/btcd/integration/rpctest"
+	"github.com/btcsuite/btcd/rpcclient"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/lightningnetwork/lnd/lntest/node"
+	"github.com/lightningnetwork/lnd/lntest/wait"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	// minerLogFilename is the default log filename for the miner node.
+	minerLogFilename = "output_btcd_miner.log"
+
+	// minerLogDir is the default log dir for the miner node.
+	minerLogDir = ".minerlogs"
+
+	// slowMineDelay defines a wait period between mining new blocks.
+	slowMineDelay = 100 * time.Millisecond
+)
+
+var harnessNetParams = &chaincfg.RegressionNetParams
+
+type HarnessMiner struct {
+	*testing.T
+	*rpctest.Harness
+
+	// runCtx is a context with cancel method. It's used to signal when the
+	// node needs to quit, and used as the parent context when spawning
+	runCtx context.Context
+	cancel context.CancelFunc
+
+	// logPath is the directory path of the miner's logs.
+	logPath string
+
+	// logFilename is the saved log filename of the miner node.
+	logFilename string
+}
+
+// NewMiner creates a new miner using btcd backend with the default log file
+// dir and name.
+func NewMiner(ctxt context.Context, t *testing.T) *HarnessMiner {
+	return newMiner(ctxt, t, minerLogDir, minerLogFilename)
+}
+
+// NewTempMiner creates a new miner using btcd backend with the specified log
+// file dir and name.
+func NewTempMiner(ctxt context.Context, t *testing.T,
+	tempDir, tempLogFilename string) *HarnessMiner {
+
+	t.Helper()
+
+	return newMiner(ctxt, t, tempDir, tempLogFilename)
+}
+
+// newMiner creates a new miner using btcd's rpctest.
+func newMiner(ctxb context.Context, t *testing.T, minerDirName,
+	logFilename string) *HarnessMiner {
+
+	handler := &rpcclient.NotificationHandlers{}
+	btcdBinary := node.GetBtcdBinary()
+	baseLogPath := fmt.Sprintf("%s/%s", node.GetLogDir(), minerDirName)
+
+	args := []string{
+		"--rejectnonstd",
+		"--txindex",
+		"--nowinservice",
+		"--nobanning",
+		"--debuglevel=debug",
+		"--logdir=" + baseLogPath,
+		"--trickleinterval=100ms",
+		// Don't disconnect if a reply takes too long.
+		"--nostalldetect",
+	}
+
+	miner, err := rpctest.New(harnessNetParams, handler, args, btcdBinary)
+	require.NoError(t, err, "unable to create mining node")
+
+	ctxt, cancel := context.WithCancel(ctxb)
+	return &HarnessMiner{
+		T:           t,
+		Harness:     miner,
+		runCtx:      ctxt,
+		cancel:      cancel,
+		logPath:     baseLogPath,
+		logFilename: logFilename,
+	}
+}
+
+// saveLogs copies the node logs and save it to the file specified by
+// h.logFilename.
+func (h *HarnessMiner) saveLogs() {
+	// After shutting down the miner, we'll make a copy of the log files
+	// before deleting the temporary log dir.
+	path := fmt.Sprintf("%s/%s", h.logPath, harnessNetParams.Name)
+	files, err := ioutil.ReadDir(path)
+	require.NoError(h, err, "unable to read log directory")
+
+	for _, file := range files {
+		newFilename := strings.Replace(
+			file.Name(), "btcd.log", h.logFilename, 1,
+		)
+		copyPath := fmt.Sprintf("%s/../%s", h.logPath, newFilename)
+
+		logFile := fmt.Sprintf("%s/%s", path, file.Name())
+		err := CopyFile(filepath.Clean(copyPath), logFile)
+		require.NoError(h, err, "unable to copy file")
+	}
+
+	err = os.RemoveAll(h.logPath)
+	require.NoErrorf(h, err, "cannot remove dir %s", h.logPath)
+}
+
+// Stop shuts down the miner and saves its logs.
+func (h *HarnessMiner) Stop() {
+	h.cancel()
+	require.NoError(h, h.TearDown(), "tear down miner got error")
+	h.saveLogs()
+}
+
+// GetBestBlock makes a RPC request to miner and asserts.
+func (h *HarnessMiner) GetBestBlock() (*chainhash.Hash, int32) {
+	blockHash, height, err := h.Client.GetBestBlock()
+	require.NoError(h, err, "failed to GetBestBlock")
+	return blockHash, height
+}
+
+// GetRawMempool makes a RPC call to the miner's GetRawMempool and
+// asserts.
+func (h *HarnessMiner) GetRawMempool() []*chainhash.Hash {
+	mempool, err := h.Client.GetRawMempool()
+	require.NoError(h, err, "unable to get mempool")
+	return mempool
+}
+
+// GenerateBlocks mine 'num' of blocks and returns them.
+func (h *HarnessMiner) GenerateBlocks(num uint32) []*chainhash.Hash {
+	blockHashes, err := h.Client.Generate(num)
+	require.NoError(h, err, "unable to generate blocks")
+	require.Len(h, blockHashes, int(num), "wrong num of blocks generated")
+
+	return blockHashes
+}
+
+// GetBlock gets a block using its block hash.
+func (h *HarnessMiner) GetBlock(blockHash *chainhash.Hash) *wire.MsgBlock {
+	block, err := h.Client.GetBlock(blockHash)
+	require.NoError(h, err, "unable to get block")
+	return block
+}
+
+// MineBlocks mine 'num' of blocks and check that blocks are present in
+// node blockchain.
+func (h *HarnessMiner) MineBlocks(num uint32) []*wire.MsgBlock {
+	blocks := make([]*wire.MsgBlock, num)
+
+	blockHashes := h.GenerateBlocks(num)
+
+	for i, blockHash := range blockHashes {
+		block := h.GetBlock(blockHash)
+		blocks[i] = block
+	}
+
+	return blocks
+}
+
+// AssertNumTxsInMempool polls until finding the desired number of transactions
+// in the provided miner's mempool. It will asserrt if this number is not met
+// after the given timeout.
+func (h *HarnessMiner) AssertNumTxsInMempool(n int) []*chainhash.Hash {
+	var (
+		mem []*chainhash.Hash
+		err error
+	)
+
+	err = wait.NoError(func() error {
+		// We require the RPC call to be succeeded and won't wait for
+		// it as it's an unexpected behavior.
+		mem = h.GetRawMempool()
+		if len(mem) == n {
+			return nil
+		}
+
+		return fmt.Errorf("want %v, got %v in mempool: %v",
+			n, len(mem), mem)
+	}, wait.MinerMempoolTimeout)
+	require.NoError(h, err, "assert tx in mempool timeout")
+
+	return mem
+}
+
+// AssertTxInBlock asserts that a given txid can be found in the passed block.
+func (h *HarnessMiner) AssertTxInBlock(block *wire.MsgBlock,
+	txid *chainhash.Hash) {
+
+	blockTxes := make([]chainhash.Hash, 0)
+
+	for _, tx := range block.Transactions {
+		sha := tx.TxHash()
+		blockTxes = append(blockTxes, sha)
+
+		if bytes.Equal(txid[:], sha[:]) {
+			return
+		}
+	}
+
+	require.Failf(h, "tx was not included in block", "tx:%v, block has:%v",
+		txid, blockTxes)
+}
+
+// MineBlocksAndAssertNumTxes mine 'num' of blocks and check that blocks are
+// present in node blockchain. numTxs should be set to the number of
+// transactions (excluding the coinbase) we expect to be included in the first
+// mined block.
+func (h *HarnessMiner) MineBlocksAndAssertNumTxes(num uint32,
+	numTxs int) []*wire.MsgBlock {
+
+	// If we expect transactions to be included in the blocks we'll mine,
+	// we wait here until they are seen in the miner's mempool.
+	txids := h.AssertNumTxsInMempool(numTxs)
+
+	// Mine blocks.
+	blocks := h.MineBlocks(num)
+
+	// Finally, assert that all the transactions were included in the first
+	// block.
+	for _, txid := range txids {
+		h.AssertTxInBlock(blocks[0], txid)
+	}
+
+	return blocks
+}
+
+// GetRawTransaction makes a RPC call to the miner's GetRawTransaction and
+// asserts.
+func (h *HarnessMiner) GetRawTransaction(txid *chainhash.Hash) *btcutil.Tx {
+	tx, err := h.Client.GetRawTransaction(txid)
+	require.NoErrorf(h, err, "failed to get raw tx: %v", txid)
+	return tx
+}
+
+// GetRawTransactionVerbose makes a RPC call to the miner's
+// GetRawTransactionVerbose and asserts.
+func (h *HarnessMiner) GetRawTransactionVerbose(
+	txid *chainhash.Hash) *btcjson.TxRawResult {
+
+	tx, err := h.Client.GetRawTransactionVerbose(txid)
+	require.NoErrorf(h, err, "failed to get raw tx verbose: %v", txid)
+	return tx
+}
+
+// AssertTxInMempool asserts a given transaction can be found in the mempool.
+func (h *HarnessMiner) AssertTxInMempool(txid *chainhash.Hash) *wire.MsgTx {
+	var msgTx *wire.MsgTx
+
+	err := wait.NoError(func() error {
+		// We require the RPC call to be succeeded and won't wait for
+		// it as it's an unexpected behavior.
+		mempool := h.GetRawMempool()
+
+		if len(mempool) == 0 {
+			return fmt.Errorf("empty mempool")
+		}
+
+		for _, memTx := range mempool {
+			// Check the values are equal.
+			if *memTx == *txid {
+				return nil
+			}
+		}
+
+		return fmt.Errorf("txid %v not found in mempool: %v", txid,
+			mempool)
+	}, wait.MinerMempoolTimeout)
+
+	require.NoError(h, err, "timeout checking mempool")
+	return msgTx
+}
+
+// SendOutputsWithoutChange uses the miner to send the given outputs using the
+// specified fee rate and returns the txid.
+func (h *HarnessMiner) SendOutputsWithoutChange(outputs []*wire.TxOut,
+	feeRate btcutil.Amount) *chainhash.Hash {
+
+	txid, err := h.Harness.SendOutputsWithoutChange(
+		outputs, feeRate,
+	)
+	require.NoErrorf(h, err, "failed to send output")
+
+	return txid
+}
+
+// CreateTransaction uses the miner to create a transaction using the given
+// outputs using the specified fee rate and returns the transaction.
+func (h *HarnessMiner) CreateTransaction(outputs []*wire.TxOut,
+	feeRate btcutil.Amount) *wire.MsgTx {
+
+	tx, err := h.Harness.CreateTransaction(outputs, feeRate, false)
+	require.NoErrorf(h, err, "failed to create transaction")
+
+	return tx
+}
+
+// SendOutput creates, signs, and finally broadcasts a transaction spending
+// the harness' available mature coinbase outputs to create the new output.
+func (h *HarnessMiner) SendOutput(newOutput *wire.TxOut,
+	feeRate btcutil.Amount) *chainhash.Hash {
+
+	hash, err := h.Harness.SendOutputs([]*wire.TxOut{newOutput}, feeRate)
+	require.NoErrorf(h, err, "failed to send outputs")
+
+	return hash
+}
+
+// MineBlocksSlow mines 'num' of blocks. Between each mined block an artificial
+// delay is introduced to give all network participants time to catch up.
+func (h *HarnessMiner) MineBlocksSlow(num uint32) []*wire.MsgBlock {
+	blocks := make([]*wire.MsgBlock, num)
+	blockHashes := make([]*chainhash.Hash, 0, num)
+
+	for i := uint32(0); i < num; i++ {
+		generatedHashes := h.GenerateBlocks(1)
+		blockHashes = append(blockHashes, generatedHashes...)
+
+		time.Sleep(slowMineDelay)
+	}
+
+	for i, blockHash := range blockHashes {
+		block, err := h.Client.GetBlock(blockHash)
+		require.NoError(h, err, "get blocks")
+
+		blocks[i] = block
+	}
+
+	return blocks
+}
+
+// AssertOutpointInMempool asserts a given outpoint can be found in the mempool.
+func (h *HarnessMiner) AssertOutpointInMempool(op wire.OutPoint) *wire.MsgTx {
+	var msgTx *wire.MsgTx
+
+	err := wait.NoError(func() error {
+		// We require the RPC call to be succeeded and won't wait for
+		// it as it's an unexpected behavior.
+		mempool := h.GetRawMempool()
+
+		if len(mempool) == 0 {
+			return fmt.Errorf("empty mempool")
+		}
+
+		for _, txid := range mempool {
+			// We require the RPC call to be succeeded and won't
+			// wait for it as it's an unexpected behavior.
+			tx := h.GetRawTransaction(txid)
+
+			msgTx = tx.MsgTx()
+			for _, txIn := range msgTx.TxIn {
+				if txIn.PreviousOutPoint == op {
+					return nil
+				}
+			}
+		}
+
+		return fmt.Errorf("outpoint %v not found in mempool", op)
+	}, wait.MinerMempoolTimeout)
+
+	require.NoError(h, err, "timeout checking mempool")
+
+	return msgTx
+}
+
+// GetNumTxsFromMempool polls until finding the desired number of transactions
+// in the miner's mempool and returns the full transactions to the caller.
+func (h *HarnessMiner) GetNumTxsFromMempool(n int) []*wire.MsgTx {
+	txids := h.AssertNumTxsInMempool(n)
+
+	var txes []*wire.MsgTx
+	for _, txid := range txids {
+		tx := h.GetRawTransaction(txid)
+		txes = append(txes, tx.MsgTx())
+	}
+
+	return txes
+}
+
+// NewMinerAddress creates a new address for the miner and asserts.
+func (h *HarnessMiner) NewMinerAddress() btcutil.Address {
+	addr, err := h.NewAddress()
+	require.NoError(h, err, "failed to create new miner address")
+	return addr
+}
+
+// MineBlocksWithTxes mines a single block to include the specifies
+// transactions only.
+func (h *HarnessMiner) MineBlockWithTxes(txes []*btcutil.Tx) *wire.MsgBlock {
+	var emptyTime time.Time
+
+	// Generate a block.
+	b, err := h.GenerateAndSubmitBlock(txes, -1, emptyTime)
+	require.NoError(h, err, "unable to mine block")
+
+	block, err := h.Client.GetBlock(b.Hash())
+	require.NoError(h, err, "unable to get block")
+
+	return block
+}
+
+// MineEmptyBlocks mines a given number of empty blocks.
+func (h *HarnessMiner) MineEmptyBlocks(num int) []*wire.MsgBlock {
+	var emptyTime time.Time
+
+	blocks := make([]*wire.MsgBlock, num)
+	for i := 0; i < num; i++ {
+		// Generate an empty block.
+		b, err := h.GenerateAndSubmitBlock(nil, -1, emptyTime)
+		require.NoError(h, err, "unable to mine empty block")
+
+		block := h.GetBlock(b.Hash())
+		blocks[i] = block
+	}
+
+	return blocks
+}

lntest/harness_node_manager.go

@@ -0,0 +1,205 @@
+package lntest
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"github.com/lightningnetwork/lnd/lnrpc"
+	"github.com/lightningnetwork/lnd/lntest/node"
+	"github.com/lightningnetwork/lnd/lntest/wait"
+)
+
+// nodeManager is responsible for hanlding the start and stop of a given node.
+// It also keeps track of the running nodes.
+type nodeManager struct {
+	sync.Mutex
+
+	// chainBackend houses the information necessary to use a node as LND
+	// chain backend, such as rpc configuration, P2P information etc.
+	chainBackend node.BackendConfig
+
+	// currentTestCase holds the name for the currently run test case.
+	currentTestCase string
+
+	// lndBinary is the full path to the lnd binary that was specifically
+	// compiled with all required itest flags.
+	lndBinary string
+
+	// dbBackend sets the database backend to use.
+	dbBackend node.DatabaseBackend
+
+	// activeNodes is a map of all running nodes, format:
+	// {pubkey: *HarnessNode}.
+	activeNodes map[uint32]*node.HarnessNode
+
+	// standbyNodes is a map of all the standby nodes, format:
+	// {pubkey: *HarnessNode}.
+	standbyNodes map[uint32]*node.HarnessNode
+
+	// nodeCounter is a monotonically increasing counter that's used as the
+	// node's unique ID.
+	nodeCounter uint32
+
+	// feeServiceURL is the url of the fee service.
+	feeServiceURL string
+}
+
+// newNodeManager creates a new node manager instance.
+func newNodeManager(lndBinary string,
+	dbBackend node.DatabaseBackend) *nodeManager {
+
+	return &nodeManager{
+		lndBinary:    lndBinary,
+		dbBackend:    dbBackend,
+		activeNodes:  make(map[uint32]*node.HarnessNode),
+		standbyNodes: make(map[uint32]*node.HarnessNode),
+	}
+}
+
+// nextNodeID generates a unique sequence to be used as the node's ID.
+func (nm *nodeManager) nextNodeID() uint32 {
+	nodeID := atomic.AddUint32(&nm.nodeCounter, 1)
+	return nodeID - 1
+}
+
+// newNode initializes a new HarnessNode, supporting the ability to initialize
+// a wallet with or without a seed. If useSeed is false, the returned harness
+// node can be used immediately. Otherwise, the node will require an additional
+// initialization phase where the wallet is either created or restored.
+func (nm *nodeManager) newNode(t *testing.T, name string, extraArgs []string,
+	password []byte, noAuth bool) (*node.HarnessNode, error) {
+
+	cfg := &node.BaseNodeConfig{
+		Name:              name,
+		LogFilenamePrefix: nm.currentTestCase,
+		Password:          password,
+		BackendCfg:        nm.chainBackend,
+		ExtraArgs:         extraArgs,
+		FeeURL:            nm.feeServiceURL,
+		DbBackend:         nm.dbBackend,
+		NodeID:            nm.nextNodeID(),
+		LndBinary:         nm.lndBinary,
+		NetParams:         harnessNetParams,
+		SkipUnlock:        noAuth,
+	}
+
+	node, err := node.NewHarnessNode(t, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	// Put node in activeNodes to ensure Shutdown is called even if start
+	// returns an error.
+	nm.registerNode(node)
+
+	return node, nil
+}
+
+// RegisterNode records a new HarnessNode in the NetworkHarnesses map of known
+// nodes. This method should only be called with nodes that have successfully
+// retrieved their public keys via FetchNodeInfo.
+func (nm *nodeManager) registerNode(node *node.HarnessNode) {
+	nm.Lock()
+	nm.activeNodes[node.Cfg.NodeID] = node
+	nm.Unlock()
+}
+
+// ShutdownNode stops an active lnd process and returns when the process has
+// exited and any temporary directories have been cleaned up.
+func (nm *nodeManager) shutdownNode(node *node.HarnessNode) error {
+	if err := node.Shutdown(); err != nil {
+		return err
+	}
+
+	delete(nm.activeNodes, node.Cfg.NodeID)
+	return nil
+}
+
+// restartNode attempts to restart a lightning node by shutting it down
+// cleanly, then restarting the process. This function is fully blocking. Upon
+// restart, the RPC connection to the node will be re-attempted, continuing iff
+// the connection attempt is successful. If the callback parameter is non-nil,
+// then the function will be executed after the node shuts down, but *before*
+// the process has been started up again.
+func (nm *nodeManager) restartNode(ctxt context.Context,
+	hn *node.HarnessNode, callback func() error) error {
+
+	// Stop the node.
+	if err := hn.Stop(); err != nil {
+		return fmt.Errorf("restart node got error: %w", err)
+	}
+
+	if callback != nil {
+		if err := callback(); err != nil {
+			return err
+		}
+	}
+
+	// Start the node without unlocking the wallet.
+	if hn.Cfg.SkipUnlock {
+		return hn.StartWithNoAuth(ctxt)
+	}
+
+	return hn.Start(ctxt)
+}
+
+// unlockNode unlocks the node's wallet if the password is configured.
+// Additionally, each time the node is unlocked, the caller can pass a set of
+// SCBs to pass in via the Unlock method allowing them to restore channels
+// during restart.
+func (nm *nodeManager) unlockNode(hn *node.HarnessNode,
+	chanBackups ...*lnrpc.ChanBackupSnapshot) error {
+
+	// If the node doesn't have a password set, then we can exit here as we
+	// don't need to unlock it.
+	if len(hn.Cfg.Password) == 0 {
+		return nil
+	}
+
+	// Otherwise, we'll unlock the wallet, then complete the final steps
+	// for the node initialization process.
+	unlockReq := &lnrpc.UnlockWalletRequest{
+		WalletPassword: hn.Cfg.Password,
+	}
+	if len(chanBackups) != 0 {
+		unlockReq.ChannelBackups = chanBackups[0]
+		unlockReq.RecoveryWindow = 100
+	}
+
+	err := wait.NoError(func() error {
+		return hn.Unlock(unlockReq)
+	}, DefaultTimeout)
+	if err != nil {
+		return fmt.Errorf("%s: failed to unlock: %w", hn.Name(), err)
+	}
+
+	return nil
+}
+
+// initWalletAndNode will unlock the node's wallet and finish setting up the
+// node so it's ready to take RPC requests.
+func (nm *nodeManager) initWalletAndNode(hn *node.HarnessNode,
+	req *lnrpc.InitWalletRequest) ([]byte, error) {
+
+	// Pass the init request via rpc to finish unlocking the node.
+	resp := hn.RPC.InitWallet(req)
+
+	// Now that the wallet is unlocked, before creating an authed
+	// connection we will close the old unauthed connection.
+	if err := hn.CloseConn(); err != nil {
+		return nil, fmt.Errorf("close unauthed conn failed")
+	}
+
+	// Init the node, which will create the authed grpc conn and all its
+	// rpc clients.
+	err := hn.InitNode(resp.AdminMacaroon)
+
+	// In stateless initialization mode we get a macaroon back that we have
+	// to return to the test, otherwise gRPC calls won't be possible since
+	// there are no macaroon files created in that mode.
+	// In stateful init the admin macaroon will just be nil.
+	return resp.AdminMacaroon, err
+}

lntest/harness_setup.go

@@ -0,0 +1,120 @@
+package lntest
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/btcsuite/btcd/integration/rpctest"
+	"github.com/lightningnetwork/lnd/lntest/node"
+	"github.com/stretchr/testify/require"
+)
+
+// SetupHarness creates a new HarnessTest with a series of setups such that the
+// instance is ready for usage. The setups are,
+// 1. create the directories to hold lnd files.
+// 2. start a btcd miner.
+// 3. start a chain backend(btcd, bitcoind, or neutrino).
+// 4. connect the miner and the chain backend.
+// 5. start the HarnessTest.
+func SetupHarness(t *testing.T, binaryPath, dbBackendName string,
+	feeService WebFeeService) *HarnessTest {
+
+	t.Log("Setting up HarnessTest...")
+
+	// Parse testing flags that influence our test execution.
+	logDir := node.GetLogDir()
+	require.NoError(t, os.MkdirAll(logDir, 0700), "create log dir failed")
+
+	// Parse database backend
+	dbBackend := prepareDbBackend(t, dbBackendName)
+
+	// Create a new HarnessTest.
+	ht := NewHarnessTest(t, binaryPath, feeService, dbBackend)
+
+	// Init the miner.
+	t.Log("Prepare the miner and mine blocks to activate segwit...")
+	miner := prepareMiner(ht.runCtx, ht.T)
+
+	// Start a chain backend.
+	chainBackend, cleanUp := prepareChainBackend(t, miner.P2PAddress())
+	ht.stopChainBackend = cleanUp
+
+	// Connect our chainBackend to our miner.
+	t.Log("Connecting the miner with the chain backend...")
+	require.NoError(t, chainBackend.ConnectMiner(), "connect miner")
+
+	// Start the HarnessTest with the chainBackend and miner.
+	ht.Start(chainBackend, miner)
+
+	return ht
+}
+
+// prepareMiner creates an instance of the btcd's rpctest.Harness that will act
+// as the miner for all tests. This will be used to fund the wallets of the
+// nodes within the test network and to drive blockchain related events within
+// the network. Revert the default setting of accepting non-standard
+// transactions on simnet to reject them. Transactions on the lightning network
+// should always be standard to get better guarantees of getting included in to
+// blocks.
+func prepareMiner(ctxt context.Context, t *testing.T) *HarnessMiner {
+	miner := NewMiner(ctxt, t)
+
+	// Before we start anything, we want to overwrite some of the
+	// connection settings to make the tests more robust. We might need to
+	// restart the miner while there are already blocks present, which will
+	// take a bit longer than the 1 second the default settings amount to.
+	// Doubling both values will give us retries up to 4 seconds.
+	miner.MaxConnRetries = rpctest.DefaultMaxConnectionRetries * 2
+	miner.ConnectionRetryTimeout = rpctest.DefaultConnectionRetryTimeout * 2
+
+	// Set up miner and connect chain backend to it.
+	require.NoError(t, miner.SetUp(true, 50))
+	require.NoError(t, miner.Client.NotifyNewTransactions(false))
+
+	// Next mine enough blocks in order for segwit and the CSV package
+	// soft-fork to activate on SimNet.
+	numBlocks := harnessNetParams.MinerConfirmationWindow * 2
+	miner.GenerateBlocks(numBlocks)
+
+	return miner
+}
+
+// prepareChainBackend creates a new chain backend.
+func prepareChainBackend(t *testing.T,
+	minerAddr string) (node.BackendConfig, func()) {
+
+	chainBackend, cleanUp, err := NewBackend(
+		minerAddr, harnessNetParams,
+	)
+	require.NoError(t, err, "new backend")
+
+	return chainBackend, func() {
+		require.NoError(t, cleanUp(), "cleanup")
+	}
+}
+
+// prepareDbBackend parses a DatabaseBackend based on the name given.
+func prepareDbBackend(t *testing.T,
+	dbBackendName string) node.DatabaseBackend {
+
+	var dbBackend node.DatabaseBackend
+	switch dbBackendName {
+	case "bbolt":
+		dbBackend = node.BackendBbolt
+
+	case "etcd":
+		dbBackend = node.BackendEtcd
+
+	case "postgres":
+		dbBackend = node.BackendPostgres
+
+	case "sqlite":
+		dbBackend = node.BackendSqlite
+
+	default:
+		require.Fail(t, "unknown db backend")
+	}
+
+	return dbBackend
+}

lntest/neutrino.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 
 	"github.com/btcsuite/btcd/chaincfg"
-	"github.com/lightningnetwork/lnd/lntemp/node"
+	"github.com/lightningnetwork/lnd/lntest/node"
 )
 
 // NeutrinoBackendConfig is an implementation of the BackendConfig interface

lntest/node/config.go

@@ -0,0 +1,398 @@
+package node
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path"
+	"path/filepath"
+	"sync/atomic"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/lightningnetwork/lnd/chanbackup"
+	"github.com/lightningnetwork/lnd/kvdb/etcd"
+	"github.com/lightningnetwork/lnd/lntest/wait"
+)
+
+const (
+	// ListenerFormat is the format string that is used to generate local
+	// listener addresses.
+	ListenerFormat = "127.0.0.1:%d"
+
+	// DefaultCSV is the CSV delay (remotedelay) we will start our test
+	// nodes with.
+	DefaultCSV = 4
+
+	// defaultNodePort is the start of the range for listening ports of
+	// harness nodes. Ports are monotonically increasing starting from this
+	// number and are determined by the results of NextAvailablePort().
+	defaultNodePort = 5555
+)
+
+var (
+	// lastPort is the last port determined to be free for use by a new
+	// node. It should be used atomically.
+	lastPort uint32 = defaultNodePort
+
+	// logOutput is a flag that can be set to append the output from the
+	// seed nodes to log files.
+	logOutput = flag.Bool("logoutput", false,
+		"log output from node n to file output-n.log")
+
+	// logSubDir is the default directory where the logs are written to if
+	// logOutput is true.
+	logSubDir = flag.String("logdir", ".", "default dir to write logs to")
+
+	// btcdExecutable is the full path to the btcd binary.
+	btcdExecutable = flag.String(
+		"btcdexec", "", "full path to btcd binary",
+	)
+)
+
+type DatabaseBackend int
+
+const (
+	BackendBbolt DatabaseBackend = iota
+	BackendEtcd
+	BackendPostgres
+	BackendSqlite
+)
+
+// Option is a function for updating a node's configuration.
+type Option func(*BaseNodeConfig)
+
+// BackendConfig is an interface that abstracts away the specific chain backend
+// node implementation.
+type BackendConfig interface {
+	// GenArgs returns the arguments needed to be passed to LND at startup
+	// for using this node as a chain backend.
+	GenArgs() []string
+
+	// ConnectMiner is called to establish a connection to the test miner.
+	ConnectMiner() error
+
+	// DisconnectMiner is called to disconnect the miner.
+	DisconnectMiner() error
+
+	// Name returns the name of the backend type.
+	Name() string
+
+	// Credentials returns the rpc username, password and host for the
+	// backend.
+	Credentials() (string, string, string, error)
+}
+
+// BaseNodeConfig is the base node configuration.
+type BaseNodeConfig struct {
+	Name string
+
+	// LogFilenamePrefix is used to prefix node log files. Can be used to
+	// store the current test case for simpler postmortem debugging.
+	LogFilenamePrefix string
+
+	NetParams         *chaincfg.Params
+	BackendCfg        BackendConfig
+	BaseDir           string
+	ExtraArgs         []string
+	OriginalExtraArgs []string
+
+	DataDir        string
+	LogDir         string
+	TLSCertPath    string
+	TLSKeyPath     string
+	AdminMacPath   string
+	ReadMacPath    string
+	InvoiceMacPath string
+
+	SkipUnlock bool
+	Password   []byte
+
+	P2PPort     int
+	RPCPort     int
+	RESTPort    int
+	ProfilePort int
+
+	FeeURL string
+
+	DbBackend   DatabaseBackend
+	PostgresDsn string
+
+	// NodeID is a unique ID used to identify the node.
+	NodeID uint32
+
+	// LndBinary is the full path to the lnd binary that was specifically
+	// compiled with all required itest flags.
+	LndBinary string
+
+	// backupDbDir is the path where a database backup is stored, if any.
+	backupDbDir string
+
+	// postgresDbName is the name of the postgres database where lnd data
+	// is stored in.
+	postgresDbName string
+}
+
+func (cfg BaseNodeConfig) P2PAddr() string {
+	return fmt.Sprintf(ListenerFormat, cfg.P2PPort)
+}
+
+func (cfg BaseNodeConfig) RPCAddr() string {
+	return fmt.Sprintf(ListenerFormat, cfg.RPCPort)
+}
+
+func (cfg BaseNodeConfig) RESTAddr() string {
+	return fmt.Sprintf(ListenerFormat, cfg.RESTPort)
+}
+
+// DBDir returns the holding directory path of the graph database.
+func (cfg BaseNodeConfig) DBDir() string {
+	return filepath.Join(cfg.DataDir, "graph", cfg.NetParams.Name)
+}
+
+func (cfg BaseNodeConfig) DBPath() string {
+	return filepath.Join(cfg.DBDir(), "channel.db")
+}
+
+func (cfg BaseNodeConfig) ChanBackupPath() string {
+	return filepath.Join(
+		cfg.DataDir, "chain", "bitcoin",
+		fmt.Sprintf(
+			"%v/%v", cfg.NetParams.Name,
+			chanbackup.DefaultBackupFileName,
+		),
+	)
+}
+
+// GenerateListeningPorts generates the ports to listen on designated for the
+// current lightning network test.
+func (cfg *BaseNodeConfig) GenerateListeningPorts() {
+	if cfg.P2PPort == 0 {
+		cfg.P2PPort = NextAvailablePort()
+	}
+	if cfg.RPCPort == 0 {
+		cfg.RPCPort = NextAvailablePort()
+	}
+	if cfg.RESTPort == 0 {
+		cfg.RESTPort = NextAvailablePort()
+	}
+	if cfg.ProfilePort == 0 {
+		cfg.ProfilePort = NextAvailablePort()
+	}
+}
+
+// BaseConfig returns the base node configuration struct.
+func (cfg *BaseNodeConfig) BaseConfig() *BaseNodeConfig {
+	return cfg
+}
+
+// GenArgs generates a slice of command line arguments from the lightning node
+// config struct.
+func (cfg *BaseNodeConfig) GenArgs() []string {
+	var args []string
+
+	switch cfg.NetParams {
+	case &chaincfg.TestNet3Params:
+		args = append(args, "--bitcoin.testnet")
+	case &chaincfg.SimNetParams:
+		args = append(args, "--bitcoin.simnet")
+	case &chaincfg.RegressionNetParams:
+		args = append(args, "--bitcoin.regtest")
+	}
+
+	backendArgs := cfg.BackendCfg.GenArgs()
+	args = append(args, backendArgs...)
+
+	nodeArgs := []string{
+		"--bitcoin.active",
+		"--nobootstrap",
+		"--debuglevel=debug",
+		"--bitcoin.defaultchanconfs=1",
+		"--accept-keysend",
+		"--keep-failed-payment-attempts",
+		fmt.Sprintf("--db.batch-commit-interval=%v", commitInterval),
+		fmt.Sprintf("--bitcoin.defaultremotedelay=%v", DefaultCSV),
+		fmt.Sprintf("--rpclisten=%v", cfg.RPCAddr()),
+		fmt.Sprintf("--restlisten=%v", cfg.RESTAddr()),
+		fmt.Sprintf("--restcors=https://%v", cfg.RESTAddr()),
+		fmt.Sprintf("--listen=%v", cfg.P2PAddr()),
+		fmt.Sprintf("--externalip=%v", cfg.P2PAddr()),
+		fmt.Sprintf("--lnddir=%v", cfg.BaseDir),
+		fmt.Sprintf("--adminmacaroonpath=%v", cfg.AdminMacPath),
+		fmt.Sprintf("--readonlymacaroonpath=%v", cfg.ReadMacPath),
+		fmt.Sprintf("--invoicemacaroonpath=%v", cfg.InvoiceMacPath),
+		fmt.Sprintf("--trickledelay=%v", trickleDelay),
+		fmt.Sprintf("--profile=%d", cfg.ProfilePort),
+		fmt.Sprintf("--caches.rpc-graph-cache-duration=%d", 0),
+
+		// Use a small batch window so we can broadcast our sweep
+		// transactions faster.
+		"--sweeper.batchwindowduration=5s",
+
+		// Use a small batch delay so we can broadcast the
+		// announcements quickly in the tests.
+		"--gossip.sub-batch-delay=5ms",
+	}
+	args = append(args, nodeArgs...)
+
+	if cfg.Password == nil {
+		args = append(args, "--noseedbackup")
+	}
+
+	switch cfg.DbBackend {
+	case BackendEtcd:
+		args = append(args, "--db.backend=etcd")
+		args = append(args, "--db.etcd.embedded")
+		args = append(
+			args, fmt.Sprintf(
+				"--db.etcd.embedded_client_port=%v",
+				NextAvailablePort(),
+			),
+		)
+		args = append(
+			args, fmt.Sprintf(
+				"--db.etcd.embedded_peer_port=%v",
+				NextAvailablePort(),
+			),
+		)
+		args = append(
+			args, fmt.Sprintf(
+				"--db.etcd.embedded_log_file=%v",
+				path.Join(cfg.LogDir, "etcd.log"),
+			),
+		)
+
+	case BackendPostgres:
+		args = append(args, "--db.backend=postgres")
+		args = append(args, "--db.postgres.dsn="+cfg.PostgresDsn)
+
+	case BackendSqlite:
+		args = append(args, "--db.backend=sqlite")
+		args = append(args, fmt.Sprintf("--db.sqlite.busytimeout=%v",
+			wait.SqliteBusyTimeout))
+	}
+
+	if cfg.FeeURL != "" {
+		args = append(args, "--feeurl="+cfg.FeeURL)
+	}
+
+	// Put extra args in the end so the args can be overwritten.
+	if cfg.ExtraArgs != nil {
+		args = append(args, cfg.ExtraArgs...)
+	}
+
+	return args
+}
+
+// ExtraArgsEtcd returns extra args for configuring LND to use an external etcd
+// database (for remote channel DB and wallet DB).
+func ExtraArgsEtcd(etcdCfg *etcd.Config, name string, cluster bool,
+	leaderSessionTTL int) []string {
+
+	extraArgs := []string{
+		"--db.backend=etcd",
+		fmt.Sprintf("--db.etcd.host=%v", etcdCfg.Host),
+		fmt.Sprintf("--db.etcd.user=%v", etcdCfg.User),
+		fmt.Sprintf("--db.etcd.pass=%v", etcdCfg.Pass),
+		fmt.Sprintf("--db.etcd.namespace=%v", etcdCfg.Namespace),
+	}
+
+	if etcdCfg.InsecureSkipVerify {
+		extraArgs = append(extraArgs, "--db.etcd.insecure_skip_verify")
+	}
+
+	if cluster {
+		clusterArgs := []string{
+			"--cluster.enable-leader-election",
+			fmt.Sprintf("--cluster.id=%v", name),
+			fmt.Sprintf("--cluster.leader-session-ttl=%v",
+				leaderSessionTTL),
+		}
+		extraArgs = append(extraArgs, clusterArgs...)
+	}
+
+	return extraArgs
+}
+
+// NextAvailablePort returns the first port that is available for listening by
+// a new node. It panics if no port is found and the maximum available TCP port
+// is reached.
+func NextAvailablePort() int {
+	port := atomic.AddUint32(&lastPort, 1)
+	for port < 65535 {
+		// If there are no errors while attempting to listen on this
+		// port, close the socket and return it as available. While it
+		// could be the case that some other process picks up this port
+		// between the time the socket is closed and it's reopened in
+		// the harness node, in practice in CI servers this seems much
+		// less likely than simply some other process already being
+		// bound at the start of the tests.
+		addr := fmt.Sprintf(ListenerFormat, port)
+		l, err := net.Listen("tcp4", addr)
+		if err == nil {
+			err := l.Close()
+			if err == nil {
+				return int(port)
+			}
+		}
+		port = atomic.AddUint32(&lastPort, 1)
+	}
+
+	// No ports available? Must be a mistake.
+	panic("no ports available for listening")
+}
+
+// GetLogDir returns the passed --logdir flag or the default value if it wasn't
+// set.
+func GetLogDir() string {
+	if logSubDir != nil && *logSubDir != "" {
+		return *logSubDir
+	}
+	return "."
+}
+
+// CopyFile copies the file src to dest.
+func CopyFile(dest, src string) error {
+	s, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	d, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(d, s); err != nil {
+		d.Close()
+		return err
+	}
+
+	return d.Close()
+}
+
+// GetBtcdBinary returns the full path to the binary of the custom built btcd
+// executable or an empty string if none is set.
+func GetBtcdBinary() string {
+	if btcdExecutable != nil {
+		return *btcdExecutable
+	}
+
+	return ""
+}
+
+// GenerateBtcdListenerAddresses is a function that returns two listener
+// addresses with unique ports and should be used to overwrite rpctest's
+// default generator which is prone to use colliding ports.
+func GenerateBtcdListenerAddresses() (string, string) {
+	return fmt.Sprintf(ListenerFormat, NextAvailablePort()),
+		fmt.Sprintf(ListenerFormat, NextAvailablePort())
+}
+
+// ApplyPortOffset adds the given offset to the lastPort variable, making it
+// possible to run the tests in parallel without colliding on the same ports.
+func ApplyPortOffset(offset uint32) {
+	_ = atomic.AddUint32(&lastPort, offset)
+}

lntest/node/state.go

@@ -0,0 +1,365 @@
+package node
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"time"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/lightningnetwork/lnd/lnrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/walletrpc"
+	"github.com/lightningnetwork/lnd/lntest/rpc"
+	"github.com/lightningnetwork/lnd/lnutils"
+)
+
+type (
+	// PolicyUpdate defines a type to store channel policy updates for a
+	// given advertisingNode. It has the format,
+	// {"advertisingNode": [policy1, policy2, ...]}.
+	PolicyUpdate map[string][]*PolicyUpdateInfo
+
+	// policyUpdateMap defines a type to store channel policy updates. It
+	// has the format,
+	// {
+	//  "chanPoint1": {
+	//       "advertisingNode1": [
+	//              policy1, policy2, ...
+	//       ],
+	//       "advertisingNode2": [
+	//              policy1, policy2, ...
+	//       ]
+	//  },
+	//  "chanPoint2": ...
+	// }.
+	policyUpdateMap map[string]map[string][]*lnrpc.RoutingPolicy
+)
+
+// PolicyUpdateInfo stores the RoutingPolicy plus the connecting node info.
+type PolicyUpdateInfo struct {
+	*lnrpc.RoutingPolicy
+
+	// ConnectingNode specifies the node that is connected with the
+	// advertising node.
+	ConnectingNode string `json:"connecting_node"`
+
+	// Timestamp records the time the policy update is made.
+	Timestamp time.Time `json:"timestamp"`
+}
+
+// OpenChannelUpdate stores the open channel updates.
+type OpenChannelUpdate struct {
+	// AdvertisingNode specifies the node that advertised this update.
+	AdvertisingNode string `json:"advertising_node"`
+
+	// ConnectingNode specifies the node that is connected with the
+	// advertising node.
+	ConnectingNode string `json:"connecting_node"`
+
+	// Timestamp records the time the policy update is made.
+	Timestamp time.Time `json:"timestamp"`
+}
+
+// openChannelCount stores the total number of channel related counts.
+type openChannelCount struct {
+	Active     int
+	Inactive   int
+	Pending    int
+	Public     int
+	Private    int
+	NumUpdates uint64
+}
+
+// closedChannelCount stores the total number of closed, waiting and pending
+// force close channels.
+type closedChannelCount struct {
+	PendingForceClose int
+	WaitingClose      int
+	Closed            int
+}
+
+// utxoCount counts the total confirmed and unconfirmed UTXOs.
+type utxoCount struct {
+	Confirmed   int
+	Unconfirmed int
+}
+
+// edgeCount counts the total and public edges.
+type edgeCount struct {
+	Total  int
+	Public int
+}
+
+// paymentCount counts the complete(settled/failed) and incomplete payments.
+type paymentCount struct {
+	Total           int
+	Completed       int
+	LastIndexOffset uint64
+}
+
+// invoiceCount counts the complete(settled/failed) and incomplete invoices.
+type invoiceCount struct {
+	Total           int
+	Completed       int
+	LastIndexOffset uint64
+}
+
+// walletBalance provides a summary over balances related the node's wallet.
+type walletBalance struct {
+	TotalBalance       int64
+	ConfirmedBalance   int64
+	UnconfirmedBalance int64
+	AccountBalance     map[string]*lnrpc.WalletAccountBalance
+}
+
+// State records the current state for a given node. It provides a simple count
+// over the node so that the test can track its state. For a channel-specific
+// state check, use dedicated function to query the channel as each channel is
+// meant to be unique.
+type State struct {
+	// rpc is the RPC clients used for the current node.
+	rpc *rpc.HarnessRPC
+
+	// OpenChannel gives the summary of open channel related counts.
+	OpenChannel openChannelCount
+
+	// CloseChannel gives the summary of close channel related counts.
+	CloseChannel closedChannelCount
+
+	// Wallet gives the summary of the wallet balance.
+	Wallet walletBalance
+
+	// HTLC counts the total active HTLCs.
+	HTLC int
+
+	// Edge counts the total private/public edges.
+	Edge edgeCount
+
+	// ChannelUpdate counts the total channel updates seen from the graph
+	// subscription.
+	ChannelUpdate int
+
+	// NodeUpdate counts the total node announcements seen from the graph
+	// subscription.
+	NodeUpdate int
+
+	// UTXO counts the total active UTXOs.
+	UTXO utxoCount
+
+	// Payment counts the total payment of the node.
+	Payment paymentCount
+
+	// Invoice counts the total invoices made by the node.
+	Invoice invoiceCount
+
+	// openChans records each opened channel and how many times it has
+	// heard the announcements from its graph subscription.
+	openChans *lnutils.SyncMap[wire.OutPoint, []*OpenChannelUpdate]
+
+	// closedChans records each closed channel and its close channel update
+	// message received from its graph subscription.
+	closedChans *lnutils.SyncMap[wire.OutPoint, *lnrpc.ClosedChannelUpdate]
+
+	// numChanUpdates records the number of channel updates seen by each
+	// channel.
+	numChanUpdates *lnutils.SyncMap[wire.OutPoint, int]
+
+	// nodeUpdates records the node announcements seen by each node.
+	nodeUpdates *lnutils.SyncMap[string, []*lnrpc.NodeUpdate]
+
+	// policyUpdates defines a type to store channel policy updates. It has
+	// the format,
+	// {
+	//  "chanPoint1": {
+	//       "advertisingNode1": [
+	//              policy1, policy2, ...
+	//       ],
+	//       "advertisingNode2": [
+	//              policy1, policy2, ...
+	//       ]
+	//  },
+	//  "chanPoint2": ...
+	// }
+	policyUpdates *lnutils.SyncMap[wire.OutPoint, PolicyUpdate]
+}
+
+// newState initialize a new state with every field being set to its zero
+// value.
+func newState(rpc *rpc.HarnessRPC) *State {
+	return &State{
+		rpc: rpc,
+		openChans: &lnutils.SyncMap[
+			wire.OutPoint, []*OpenChannelUpdate,
+		]{},
+		closedChans: &lnutils.SyncMap[
+			wire.OutPoint, *lnrpc.ClosedChannelUpdate,
+		]{},
+		numChanUpdates: &lnutils.SyncMap[wire.OutPoint, int]{},
+		nodeUpdates:    &lnutils.SyncMap[string, []*lnrpc.NodeUpdate]{},
+		policyUpdates:  &lnutils.SyncMap[wire.OutPoint, PolicyUpdate]{},
+	}
+}
+
+// updateChannelStats gives the stats on open channel related fields.
+func (s *State) updateChannelStats() {
+	req := &lnrpc.ListChannelsRequest{}
+	resp := s.rpc.ListChannels(req)
+
+	for _, channel := range resp.Channels {
+		if channel.Active {
+			s.OpenChannel.Active++
+		} else {
+			s.OpenChannel.Inactive++
+		}
+
+		if channel.Private {
+			s.OpenChannel.Private++
+		} else {
+			s.OpenChannel.Public++
+		}
+		s.OpenChannel.NumUpdates += channel.NumUpdates
+		s.HTLC += len(channel.PendingHtlcs)
+	}
+}
+
+// updateCloseChannelStats gives the stats on close channel related fields.
+func (s *State) updateCloseChannelStats() {
+	resp := s.rpc.PendingChannels()
+	s.CloseChannel.PendingForceClose += len(
+		resp.PendingForceClosingChannels,
+	)
+	s.CloseChannel.WaitingClose += len(resp.WaitingCloseChannels)
+
+	closeReq := &lnrpc.ClosedChannelsRequest{}
+	closed := s.rpc.ClosedChannels(closeReq)
+
+	s.CloseChannel.Closed += len(closed.Channels)
+	s.OpenChannel.Pending += len(resp.PendingOpenChannels)
+}
+
+// updatePaymentStats counts the total payments made.
+func (s *State) updatePaymentStats() {
+	req := &lnrpc.ListPaymentsRequest{
+		IndexOffset: s.Payment.LastIndexOffset,
+	}
+	resp := s.rpc.ListPayments(req)
+
+	// Exit early when the there's no payment.
+	//
+	// NOTE: we need to exit early here because when there's no invoice the
+	// `LastOffsetIndex` will be zero.
+	if len(resp.Payments) == 0 {
+		return
+	}
+
+	s.Payment.LastIndexOffset = resp.LastIndexOffset
+	for _, payment := range resp.Payments {
+		if payment.Status == lnrpc.Payment_FAILED ||
+			payment.Status == lnrpc.Payment_SUCCEEDED {
+
+			s.Payment.Completed++
+		}
+	}
+
+	s.Payment.Total += len(resp.Payments)
+}
+
+// updateInvoiceStats counts the total invoices made.
+func (s *State) updateInvoiceStats() {
+	req := &lnrpc.ListInvoiceRequest{
+		NumMaxInvoices: math.MaxUint64,
+		IndexOffset:    s.Invoice.LastIndexOffset,
+	}
+	resp := s.rpc.ListInvoices(req)
+
+	// Exit early when the there's no invoice.
+	//
+	// NOTE: we need to exit early here because when there's no invoice the
+	// `LastOffsetIndex` will be zero.
+	if len(resp.Invoices) == 0 {
+		return
+	}
+
+	s.Invoice.LastIndexOffset = resp.LastIndexOffset
+	for _, invoice := range resp.Invoices {
+		if invoice.State == lnrpc.Invoice_SETTLED ||
+			invoice.State == lnrpc.Invoice_CANCELED {
+
+			s.Invoice.Completed++
+		}
+	}
+
+	s.Invoice.Total += len(resp.Invoices)
+}
+
+// updateUTXOStats counts the total UTXOs made.
+func (s *State) updateUTXOStats() {
+	req := &walletrpc.ListUnspentRequest{}
+	resp := s.rpc.ListUnspent(req)
+
+	for _, utxo := range resp.Utxos {
+		if utxo.Confirmations > 0 {
+			s.UTXO.Confirmed++
+		} else {
+			s.UTXO.Unconfirmed++
+		}
+	}
+}
+
+// updateEdgeStats counts the total edges.
+func (s *State) updateEdgeStats() {
+	req := &lnrpc.ChannelGraphRequest{IncludeUnannounced: true}
+	resp := s.rpc.DescribeGraph(req)
+	s.Edge.Total = len(resp.Edges)
+
+	req = &lnrpc.ChannelGraphRequest{IncludeUnannounced: false}
+	resp = s.rpc.DescribeGraph(req)
+	s.Edge.Public = len(resp.Edges)
+}
+
+// updateWalletBalance creates stats for the node's wallet balance.
+func (s *State) updateWalletBalance() {
+	resp := s.rpc.WalletBalance()
+
+	s.Wallet.TotalBalance = resp.TotalBalance
+	s.Wallet.ConfirmedBalance = resp.ConfirmedBalance
+	s.Wallet.UnconfirmedBalance = resp.UnconfirmedBalance
+	s.Wallet.AccountBalance = resp.AccountBalance
+}
+
+// updateState updates the internal state of the node.
+func (s *State) updateState() {
+	s.updateChannelStats()
+	s.updateCloseChannelStats()
+	s.updatePaymentStats()
+	s.updateInvoiceStats()
+	s.updateUTXOStats()
+	s.updateEdgeStats()
+	s.updateWalletBalance()
+}
+
+// String encodes the node's state for debugging.
+func (s *State) String() string {
+	stateBytes, err := json.MarshalIndent(s, "", "\t")
+	if err != nil {
+		return fmt.Sprintf("\n encode node state with err: %v", err)
+	}
+
+	return fmt.Sprintf("\n%s", stateBytes)
+}
+
+// resetEphermalStates resets the current state with a new HarnessRPC and empty
+// private fields which are used to track state only valid for the last test.
+func (s *State) resetEphermalStates(rpc *rpc.HarnessRPC) {
+	s.rpc = rpc
+
+	// Reset ephermal states which are used to record info from finished
+	// tests.
+	s.openChans = &lnutils.SyncMap[wire.OutPoint, []*OpenChannelUpdate]{}
+	s.closedChans = &lnutils.SyncMap[
+		wire.OutPoint, *lnrpc.ClosedChannelUpdate,
+	]{}
+	s.numChanUpdates = &lnutils.SyncMap[wire.OutPoint, int]{}
+	s.nodeUpdates = &lnutils.SyncMap[string, []*lnrpc.NodeUpdate]{}
+	s.policyUpdates = &lnutils.SyncMap[wire.OutPoint, PolicyUpdate]{}
+}

lntest/node/watcher.go

@@ -0,0 +1,689 @@
+package node
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/lightningnetwork/lnd/lnrpc"
+	"github.com/lightningnetwork/lnd/lntest/rpc"
+	"github.com/lightningnetwork/lnd/lntest/wait"
+	"github.com/lightningnetwork/lnd/lnutils"
+)
+
+type chanWatchType uint8
+
+const (
+	// watchOpenChannel specifies that this is a request to watch an open
+	// channel event.
+	watchOpenChannel chanWatchType = iota
+
+	// watchCloseChannel specifies that this is a request to watch a close
+	// channel event.
+	watchCloseChannel
+
+	// watchPolicyUpdate specifies that this is a request to watch a policy
+	// update event.
+	watchPolicyUpdate
+)
+
+// chanWatchRequest is a request to the lightningNetworkWatcher to be notified
+// once it's detected within the test Lightning Network, that a channel has
+// either been added or closed.
+type chanWatchRequest struct {
+	chanPoint wire.OutPoint
+
+	chanWatchType chanWatchType
+
+	eventChan chan struct{}
+
+	advertisingNode    string
+	policy             *lnrpc.RoutingPolicy
+	includeUnannounced bool
+}
+
+// nodeWatcher is a topology watcher for a HarnessNode. It keeps track of all
+// the topology updates seen in a given node, including NodeUpdate,
+// ChannelEdgeUpdate, and ClosedChannelUpdate.
+type nodeWatcher struct {
+	// rpc is the RPC clients used for the current node.
+	rpc *rpc.HarnessRPC
+
+	// state is the node's current state.
+	state *State
+
+	// chanWatchRequests receives a request for watching a particular event
+	// for a given channel.
+	chanWatchRequests chan *chanWatchRequest
+
+	// For each outpoint, we'll track an integer which denotes the number
+	// of edges seen for that channel within the network. When this number
+	// reaches 2, then it means that both edge advertisements has
+	// propagated through the network.
+	openChanWatchers  *lnutils.SyncMap[wire.OutPoint, []chan struct{}]
+	closeChanWatchers *lnutils.SyncMap[wire.OutPoint, []chan struct{}]
+
+	wg sync.WaitGroup
+}
+
+func newNodeWatcher(rpc *rpc.HarnessRPC, state *State) *nodeWatcher {
+	return &nodeWatcher{
+		rpc:               rpc,
+		state:             state,
+		chanWatchRequests: make(chan *chanWatchRequest, 100),
+		openChanWatchers: &lnutils.SyncMap[
+			wire.OutPoint, []chan struct{},
+		]{},
+		closeChanWatchers: &lnutils.SyncMap[
+			wire.OutPoint, []chan struct{},
+		]{},
+	}
+}
+
+// GetNumChannelUpdates reads the num of channel updates inside a lock and
+// returns the value.
+func (nw *nodeWatcher) GetNumChannelUpdates(op wire.OutPoint) int {
+	result, _ := nw.state.numChanUpdates.Load(op)
+	return result
+}
+
+// GetPolicyUpdates returns the node's policyUpdates state.
+func (nw *nodeWatcher) GetPolicyUpdates(op wire.OutPoint) PolicyUpdate {
+	result, _ := nw.state.policyUpdates.Load(op)
+	return result
+}
+
+// GetNodeUpdates reads the node updates inside a lock and returns the value.
+func (nw *nodeWatcher) GetNodeUpdates(pubkey string) []*lnrpc.NodeUpdate {
+	result, _ := nw.state.nodeUpdates.Load(pubkey)
+	return result
+}
+
+// WaitForNumChannelUpdates will block until a given number of updates has been
+// seen in the node's network topology.
+func (nw *nodeWatcher) WaitForNumChannelUpdates(op wire.OutPoint,
+	expected int) error {
+
+	checkNumUpdates := func() error {
+		num := nw.GetNumChannelUpdates(op)
+		if num >= expected {
+			return nil
+		}
+
+		return fmt.Errorf("timeout waiting for num channel updates, "+
+			"want %d, got %d", expected, num)
+	}
+
+	return wait.NoError(checkNumUpdates, wait.DefaultTimeout)
+}
+
+// WaitForNumNodeUpdates will block until a given number of node updates has
+// been seen in the node's network topology.
+func (nw *nodeWatcher) WaitForNumNodeUpdates(pubkey string,
+	expected int) ([]*lnrpc.NodeUpdate, error) {
+
+	updates := make([]*lnrpc.NodeUpdate, 0)
+	checkNumUpdates := func() error {
+		updates = nw.GetNodeUpdates(pubkey)
+		num := len(updates)
+		if num >= expected {
+			return nil
+		}
+
+		return fmt.Errorf("timeout waiting for num node updates, "+
+			"want %d, got %d", expected, num)
+	}
+	err := wait.NoError(checkNumUpdates, wait.DefaultTimeout)
+
+	return updates, err
+}
+
+// WaitForChannelOpen will block until a channel with the target outpoint is
+// seen as being fully advertised within the network. A channel is considered
+// "fully advertised" once both of its directional edges has been advertised in
+// the node's network topology.
+func (nw *nodeWatcher) WaitForChannelOpen(chanPoint *lnrpc.ChannelPoint) error {
+	op := nw.rpc.MakeOutpoint(chanPoint)
+	eventChan := make(chan struct{})
+	nw.chanWatchRequests <- &chanWatchRequest{
+		chanPoint:     op,
+		eventChan:     eventChan,
+		chanWatchType: watchOpenChannel,
+	}
+
+	timer := time.After(wait.DefaultTimeout)
+	select {
+	case <-eventChan:
+		return nil
+
+	case <-timer:
+		updates, err := syncMapToJSON(&nw.state.openChans.Map)
+		if err != nil {
+			return err
+		}
+
+		return fmt.Errorf("channel:%s not heard before timeout: "+
+			"node has heard: %s", op, updates)
+	}
+}
+
+// WaitForChannelClose will block until a channel with the target outpoint is
+// seen as closed within the node's network topology. A channel is considered
+// closed once a transaction spending the funding outpoint is seen within a
+// confirmed block.
+func (nw *nodeWatcher) WaitForChannelClose(
+	chanPoint *lnrpc.ChannelPoint) (*lnrpc.ClosedChannelUpdate, error) {
+
+	op := nw.rpc.MakeOutpoint(chanPoint)
+	eventChan := make(chan struct{})
+	nw.chanWatchRequests <- &chanWatchRequest{
+		chanPoint:     op,
+		eventChan:     eventChan,
+		chanWatchType: watchCloseChannel,
+	}
+
+	timer := time.After(wait.DefaultTimeout)
+	select {
+	case <-eventChan:
+		closedChan, ok := nw.state.closedChans.Load(op)
+		if !ok {
+			return nil, fmt.Errorf("channel:%s expected to find "+
+				"a closed channel in node's state:%s", op,
+				nw.state)
+		}
+		return closedChan, nil
+
+	case <-timer:
+		return nil, fmt.Errorf("channel:%s not closed before timeout: "+
+			"%s", op, nw.state)
+	}
+}
+
+// WaitForChannelPolicyUpdate will block until a channel policy with the target
+// outpoint and advertisingNode is seen within the network.
+func (nw *nodeWatcher) WaitForChannelPolicyUpdate(
+	advertisingNode *HarnessNode, policy *lnrpc.RoutingPolicy,
+	chanPoint *lnrpc.ChannelPoint, includeUnannounced bool) error {
+
+	op := nw.rpc.MakeOutpoint(chanPoint)
+
+	ticker := time.NewTicker(wait.PollInterval)
+	timer := time.After(wait.DefaultTimeout)
+	defer ticker.Stop()
+
+	eventChan := make(chan struct{})
+	for {
+		select {
+		// Send a watch request every second.
+		case <-ticker.C:
+			// Did the event can close in the meantime? We want to
+			// avoid a "close of closed channel" panic since we're
+			// re-using the same event chan for multiple requests.
+			select {
+			case <-eventChan:
+				return nil
+			default:
+			}
+
+			nw.chanWatchRequests <- &chanWatchRequest{
+				chanPoint:          op,
+				eventChan:          eventChan,
+				chanWatchType:      watchPolicyUpdate,
+				policy:             policy,
+				advertisingNode:    advertisingNode.PubKeyStr,
+				includeUnannounced: includeUnannounced,
+			}
+
+		case <-eventChan:
+			return nil
+
+		case <-timer:
+			expected, err := json.MarshalIndent(policy, "", "\t")
+			if err != nil {
+				return fmt.Errorf("encode policy err: %v", err)
+			}
+			policies, err := syncMapToJSON(
+				&nw.state.policyUpdates.Map,
+			)
+			if err != nil {
+				return err
+			}
+
+			return fmt.Errorf("policy not updated before timeout:"+
+				"\nchannel: %v \nadvertisingNode: %s:%v"+
+				"\nwant policy:%s\nhave updates:%s", op,
+				advertisingNode.Name(),
+				advertisingNode.PubKeyStr, expected, policies)
+		}
+	}
+}
+
+// syncMapToJSON is a helper function that creates json bytes from the sync.Map
+// used in the node. Expect the sync.Map to have map[string]interface.
+func syncMapToJSON(state *sync.Map) ([]byte, error) {
+	m := map[string]interface{}{}
+	state.Range(func(k, v interface{}) bool {
+		op := k.(wire.OutPoint)
+		m[op.String()] = v
+		return true
+	})
+	policies, err := json.MarshalIndent(m, "", "\t")
+	if err != nil {
+		return nil, fmt.Errorf("encode polices err: %v", err)
+	}
+
+	return policies, nil
+}
+
+// topologyWatcher is a goroutine which is able to dispatch notifications once
+// it has been observed that a target channel has been closed or opened within
+// the network. In order to dispatch these notifications, the
+// GraphTopologySubscription client exposed as part of the gRPC interface is
+// used.
+//
+// NOTE: must be run as a goroutine.
+func (nw *nodeWatcher) topologyWatcher(ctxb context.Context,
+	started chan error) {
+
+	graphUpdates := make(chan *lnrpc.GraphTopologyUpdate)
+
+	client, err := nw.newTopologyClient(ctxb)
+	started <- err
+
+	// Exit if there's an error.
+	if err != nil {
+		return
+	}
+
+	// Start a goroutine to receive graph updates.
+	nw.wg.Add(1)
+	go func() {
+		defer nw.wg.Done()
+
+		// With the client being created, we now start receiving the
+		// updates.
+		err = nw.receiveTopologyClientStream(ctxb, client, graphUpdates)
+		if err != nil {
+			started <- fmt.Errorf("receiveTopologyClientStream "+
+				"got err: %v", err)
+		}
+	}()
+
+	for {
+		select {
+		// A new graph update has just been received, so we'll examine
+		// the current set of registered clients to see if we can
+		// dispatch any requests.
+		case graphUpdate := <-graphUpdates:
+			nw.handleChannelEdgeUpdates(graphUpdate.ChannelUpdates)
+			nw.handleClosedChannelUpdate(graphUpdate.ClosedChans)
+			nw.handleNodeUpdates(graphUpdate.NodeUpdates)
+
+		// A new watch request, has just arrived. We'll either be able
+		// to dispatch immediately, or need to add the client for
+		// processing later.
+		case watchRequest := <-nw.chanWatchRequests:
+			switch watchRequest.chanWatchType {
+			case watchOpenChannel:
+				// TODO(roasbeef): add update type also, checks
+				// for multiple of 2
+				nw.handleOpenChannelWatchRequest(watchRequest)
+
+			case watchCloseChannel:
+				nw.handleCloseChannelWatchRequest(watchRequest)
+
+			case watchPolicyUpdate:
+				nw.handlePolicyUpdateWatchRequest(watchRequest)
+			}
+
+		case <-ctxb.Done():
+			return
+		}
+	}
+}
+
+func (nw *nodeWatcher) handleNodeUpdates(updates []*lnrpc.NodeUpdate) {
+	for _, nodeUpdate := range updates {
+		nw.updateNodeStateNodeUpdates(nodeUpdate)
+	}
+}
+
+// handleChannelEdgeUpdates takes a series of channel edge updates, extracts
+// the outpoints, and saves them to harness node's internal state.
+func (nw *nodeWatcher) handleChannelEdgeUpdates(
+	updates []*lnrpc.ChannelEdgeUpdate) {
+
+	// For each new channel, we'll increment the number of edges seen by
+	// one.
+	for _, newChan := range updates {
+		op := nw.rpc.MakeOutpoint(newChan.ChanPoint)
+
+		// Update the num of channel updates.
+		nw.updateNodeStateNumChanUpdates(op)
+
+		// Update the open channels.
+		nw.updateNodeStateOpenChannel(op, newChan)
+
+		// Check whether there's a routing policy update. If so, save
+		// it to the node state.
+		if newChan.RoutingPolicy != nil {
+			nw.updateNodeStatePolicy(op, newChan)
+		}
+	}
+}
+
+// updateNodeStateNumChanUpdates updates the internal state of the node
+// regarding the num of channel update seen.
+func (nw *nodeWatcher) updateNodeStateNumChanUpdates(op wire.OutPoint) {
+	oldNum, _ := nw.state.numChanUpdates.Load(op)
+	nw.state.numChanUpdates.Store(op, oldNum+1)
+}
+
+// updateNodeStateNodeUpdates updates the internal state of the node regarding
+// the node updates seen.
+func (nw *nodeWatcher) updateNodeStateNodeUpdates(update *lnrpc.NodeUpdate) {
+	oldUpdates, _ := nw.state.nodeUpdates.Load(update.IdentityKey)
+	nw.state.nodeUpdates.Store(
+		update.IdentityKey, append(oldUpdates, update),
+	)
+}
+
+// updateNodeStateOpenChannel updates the internal state of the node regarding
+// the open channels.
+func (nw *nodeWatcher) updateNodeStateOpenChannel(op wire.OutPoint,
+	newChan *lnrpc.ChannelEdgeUpdate) {
+
+	// Load the old updates the node has heard so far.
+	updates, _ := nw.state.openChans.Load(op)
+
+	// Create a new update based on this newChan.
+	newUpdate := &OpenChannelUpdate{
+		AdvertisingNode: newChan.AdvertisingNode,
+		ConnectingNode:  newChan.ConnectingNode,
+		Timestamp:       time.Now(),
+	}
+
+	// Update the node's state.
+	updates = append(updates, newUpdate)
+	nw.state.openChans.Store(op, updates)
+
+	// For this new channel, if the number of edges seen is less
+	// than two, then the channel hasn't been fully announced yet.
+	if len(updates) < 2 {
+		return
+	}
+
+	// Otherwise, we'll notify all the registered watchers and
+	// remove the dispatched watchers.
+	watcherResult, loaded := nw.openChanWatchers.LoadAndDelete(op)
+	if !loaded {
+		return
+	}
+
+	for _, eventChan := range watcherResult {
+		close(eventChan)
+	}
+}
+
+// updateNodeStatePolicy updates the internal state of the node regarding the
+// policy updates.
+func (nw *nodeWatcher) updateNodeStatePolicy(op wire.OutPoint,
+	newChan *lnrpc.ChannelEdgeUpdate) {
+
+	// Init an empty policy map and overwrite it if the channel point can
+	// be found in the node's policyUpdates.
+	policies, ok := nw.state.policyUpdates.Load(op)
+	if !ok {
+		policies = make(PolicyUpdate)
+	}
+
+	node := newChan.AdvertisingNode
+
+	// Append the policy to the slice and update the node's state.
+	newPolicy := PolicyUpdateInfo{
+		newChan.RoutingPolicy, newChan.ConnectingNode, time.Now(),
+	}
+	policies[node] = append(policies[node], &newPolicy)
+	nw.state.policyUpdates.Store(op, policies)
+}
+
+// handleOpenChannelWatchRequest processes a watch open channel request by
+// checking the number of the edges seen for a given channel point. If the
+// number is no less than 2 then the channel is considered open. Otherwise, we
+// will attempt to find it in its channel graph. If neither can be found, the
+// request is added to a watch request list than will be handled by
+// handleChannelEdgeUpdates.
+func (nw *nodeWatcher) handleOpenChannelWatchRequest(req *chanWatchRequest) {
+	targetChan := req.chanPoint
+
+	// If this is an open request, then it can be dispatched if the number
+	// of edges seen for the channel is at least two.
+	result, _ := nw.state.openChans.Load(targetChan)
+	if len(result) >= 2 {
+		close(req.eventChan)
+		return
+	}
+
+	// Otherwise, we'll add this to the list of open channel watchers for
+	// this out point.
+	watchers, _ := nw.openChanWatchers.Load(targetChan)
+	nw.openChanWatchers.Store(
+		targetChan, append(watchers, req.eventChan),
+	)
+}
+
+// handleClosedChannelUpdate takes a series of closed channel updates, extracts
+// the outpoints, saves them to harness node's internal state, and notifies all
+// registered clients.
+func (nw *nodeWatcher) handleClosedChannelUpdate(
+	updates []*lnrpc.ClosedChannelUpdate) {
+
+	// For each channel closed, we'll mark that we've detected a channel
+	// closure while lnd was pruning the channel graph.
+	for _, closedChan := range updates {
+		op := nw.rpc.MakeOutpoint(closedChan.ChanPoint)
+
+		nw.state.closedChans.Store(op, closedChan)
+
+		// As the channel has been closed, we'll notify all register
+		// watchers.
+		watchers, loaded := nw.closeChanWatchers.LoadAndDelete(op)
+		if !loaded {
+			continue
+		}
+
+		for _, eventChan := range watchers {
+			close(eventChan)
+		}
+	}
+}
+
+// handleCloseChannelWatchRequest processes a watch close channel request by
+// checking whether the given channel point can be found in the node's internal
+// state. If not, the request is added to a watch request list than will be
+// handled by handleCloseChannelWatchRequest.
+func (nw *nodeWatcher) handleCloseChannelWatchRequest(req *chanWatchRequest) {
+	targetChan := req.chanPoint
+
+	// If this is a close request, then it can be immediately dispatched if
+	// we've already seen a channel closure for this channel.
+	if _, ok := nw.state.closedChans.Load(targetChan); ok {
+		close(req.eventChan)
+		return
+	}
+
+	// Otherwise, we'll add this to the list of close channel watchers for
+	// this out point.
+	oldWatchers, _ := nw.closeChanWatchers.Load(targetChan)
+	nw.closeChanWatchers.Store(
+		targetChan, append(oldWatchers, req.eventChan),
+	)
+}
+
+// handlePolicyUpdateWatchRequest checks that if the expected policy can be
+// found either in the node's interval state or describe graph response. If
+// found, it will signal the request by closing the event channel. Otherwise it
+// does nothing but returns nil.
+func (nw *nodeWatcher) handlePolicyUpdateWatchRequest(req *chanWatchRequest) {
+	op := req.chanPoint
+
+	var policies []*PolicyUpdateInfo
+
+	// Get a list of known policies for this chanPoint+advertisingNode
+	// combination. Start searching in the node state first.
+	policyMap, ok := nw.state.policyUpdates.Load(op)
+	if ok {
+		policies, ok = policyMap[req.advertisingNode]
+		if !ok {
+			return
+		}
+	} else {
+		// If it cannot be found in the node state, try searching it
+		// from the node's DescribeGraph.
+		policyMap := nw.getChannelPolicies(req.includeUnannounced)
+		result, ok := policyMap[op.String()][req.advertisingNode]
+		if !ok {
+			return
+		}
+		for _, policy := range result {
+			// Use empty from node to mark it being loaded from
+			// DescribeGraph.
+			policies = append(
+				policies, &PolicyUpdateInfo{
+					policy, "", time.Now(),
+				},
+			)
+		}
+	}
+
+	// Check if the latest policy is matched.
+	policy := policies[len(policies)-1]
+	if CheckChannelPolicy(policy.RoutingPolicy, req.policy) == nil {
+		close(req.eventChan)
+		return
+	}
+}
+
+type topologyClient lnrpc.Lightning_SubscribeChannelGraphClient
+
+// newTopologyClient creates a topology client.
+func (nw *nodeWatcher) newTopologyClient(
+	ctx context.Context) (topologyClient, error) {
+
+	req := &lnrpc.GraphTopologySubscription{}
+	client, err := nw.rpc.LN.SubscribeChannelGraph(ctx, req)
+	if err != nil {
+		return nil, fmt.Errorf("%s: unable to create topology client: "+
+			"%v (%s)", nw.rpc.Name, err, time.Now().String())
+	}
+
+	return client, nil
+}
+
+// receiveTopologyClientStream takes a topologyClient and receives graph
+// updates.
+//
+// NOTE: must be run as a goroutine.
+func (nw *nodeWatcher) receiveTopologyClientStream(ctxb context.Context,
+	client topologyClient,
+	receiver chan *lnrpc.GraphTopologyUpdate) error {
+
+	for {
+		update, err := client.Recv()
+
+		switch {
+		case err == nil:
+			// Good case. We will send the update to the receiver.
+
+		case strings.Contains(err.Error(), "EOF"):
+			// End of subscription stream. Do nothing and quit.
+			return nil
+
+		case strings.Contains(err.Error(), context.Canceled.Error()):
+			// End of subscription stream. Do nothing and quit.
+			return nil
+
+		default:
+			// An expected error is returned, return and leave it
+			// to be handled by the caller.
+			return fmt.Errorf("graph subscription err: %w", err)
+		}
+
+		// Send the update or quit.
+		select {
+		case receiver <- update:
+		case <-ctxb.Done():
+			return nil
+		}
+	}
+}
+
+// getChannelPolicies queries the channel graph and formats the policies into
+// the format defined in type policyUpdateMap.
+func (nw *nodeWatcher) getChannelPolicies(include bool) policyUpdateMap {
+	req := &lnrpc.ChannelGraphRequest{IncludeUnannounced: include}
+	graph := nw.rpc.DescribeGraph(req)
+
+	policyUpdates := policyUpdateMap{}
+
+	for _, e := range graph.Edges {
+		policies := policyUpdates[e.ChanPoint]
+
+		// If the map[op] is nil, we need to initialize the map first.
+		if policies == nil {
+			policies = make(map[string][]*lnrpc.RoutingPolicy)
+		}
+
+		if e.Node1Policy != nil {
+			policies[e.Node1Pub] = append(
+				policies[e.Node1Pub], e.Node1Policy,
+			)
+		}
+
+		if e.Node2Policy != nil {
+			policies[e.Node2Pub] = append(
+				policies[e.Node2Pub], e.Node2Policy,
+			)
+		}
+
+		policyUpdates[e.ChanPoint] = policies
+	}
+
+	return policyUpdates
+}
+
+// CheckChannelPolicy checks that the policy matches the expected one.
+func CheckChannelPolicy(policy, expectedPolicy *lnrpc.RoutingPolicy) error {
+	if policy.FeeBaseMsat != expectedPolicy.FeeBaseMsat {
+		return fmt.Errorf("expected base fee %v, got %v",
+			expectedPolicy.FeeBaseMsat, policy.FeeBaseMsat)
+	}
+	if policy.FeeRateMilliMsat != expectedPolicy.FeeRateMilliMsat {
+		return fmt.Errorf("expected fee rate %v, got %v",
+			expectedPolicy.FeeRateMilliMsat,
+			policy.FeeRateMilliMsat)
+	}
+	if policy.TimeLockDelta != expectedPolicy.TimeLockDelta {
+		return fmt.Errorf("expected time lock delta %v, got %v",
+			expectedPolicy.TimeLockDelta,
+			policy.TimeLockDelta)
+	}
+	if policy.MinHtlc != expectedPolicy.MinHtlc {
+		return fmt.Errorf("expected min htlc %v, got %v",
+			expectedPolicy.MinHtlc, policy.MinHtlc)
+	}
+	if policy.MaxHtlcMsat != expectedPolicy.MaxHtlcMsat {
+		return fmt.Errorf("expected max htlc %v, got %v",
+			expectedPolicy.MaxHtlcMsat, policy.MaxHtlcMsat)
+	}
+	if policy.Disabled != expectedPolicy.Disabled {
+		return errors.New("edge should be disabled but isn't")
+	}
+
+	return nil
+}

lntest/rpc/chain_kit.go

@@ -0,0 +1,64 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/lightningnetwork/lnd/lnrpc/chainrpc"
+)
+
+// =====================
+// ChainKitClient related RPCs.
+// =====================
+
+// GetBestBlock makes an RPC call to chain kit client's GetBestBlock and
+// asserts.
+func (h *HarnessRPC) GetBestBlock(
+	req *chainrpc.GetBestBlockRequest) *chainrpc.GetBestBlockResponse {
+
+	if req == nil {
+		req = &chainrpc.GetBestBlockRequest{}
+	}
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.ChainKit.GetBestBlock(ctxt, req)
+	h.NoError(err, "GetBestBlock")
+
+	return resp
+}
+
+// GetBlock makes an RPC call to chain kit client's GetBlock and asserts.
+func (h *HarnessRPC) GetBlock(
+	req *chainrpc.GetBlockRequest) *chainrpc.GetBlockResponse {
+
+	if req == nil {
+		req = &chainrpc.GetBlockRequest{}
+	}
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.ChainKit.GetBlock(ctxt, req)
+	h.NoError(err, "GetBlock")
+
+	return resp
+}
+
+// GetBlockHash makes an RPC call to chain kit client's GetBlockHash and
+// asserts.
+func (h *HarnessRPC) GetBlockHash(
+	req *chainrpc.GetBlockHashRequest) *chainrpc.GetBlockHashResponse {
+
+	if req == nil {
+		req = &chainrpc.GetBlockHashRequest{}
+	}
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.ChainKit.GetBlockHash(ctxt, req)
+	h.NoError(err, "GetBlockHash")
+
+	return resp
+}

lntest/rpc/chain_notifier.go

@@ -0,0 +1,44 @@
+package rpc
+
+import (
+	"github.com/lightningnetwork/lnd/lnrpc/chainrpc"
+)
+
+// =====================
+// ChainClient related RPCs.
+// =====================
+
+type ConfNtfnClient chainrpc.ChainNotifier_RegisterConfirmationsNtfnClient
+
+// RegisterConfirmationsNtfn creates a notification client to watch a given
+// transaction being confirmed.
+func (h *HarnessRPC) RegisterConfirmationsNtfn(
+	req *chainrpc.ConfRequest) ConfNtfnClient {
+
+	// RegisterConfirmationsNtfn needs to have the context alive for the
+	// entire test case as the returned client will be used for send and
+	// receive events stream. Thus we use runCtx here instead of a timeout
+	// context.
+	client, err := h.ChainClient.RegisterConfirmationsNtfn(
+		h.runCtx, req,
+	)
+	h.NoError(err, "RegisterConfirmationsNtfn")
+
+	return client
+}
+
+type SpendClient chainrpc.ChainNotifier_RegisterSpendNtfnClient
+
+// RegisterSpendNtfn creates a notification client to watch a given
+// transaction being spent.
+func (h *HarnessRPC) RegisterSpendNtfn(req *chainrpc.SpendRequest) SpendClient {
+	// RegisterSpendNtfn needs to have the context alive for the entire
+	// test case as the returned client will be used for send and receive
+	// events stream. Thus we use runCtx here instead of a timeout context.
+	client, err := h.ChainClient.RegisterSpendNtfn(
+		h.runCtx, req,
+	)
+	h.NoError(err, "RegisterSpendNtfn")
+
+	return client
+}

lntest/rpc/harness_rpc.go

@@ -0,0 +1,98 @@
+package rpc
+
+import (
+	"context"
+	"testing"
+
+	"github.com/btcsuite/btcd/wire"
+	"github.com/lightningnetwork/lnd/lnrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/chainrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/invoicesrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/neutrinorpc"
+	"github.com/lightningnetwork/lnd/lnrpc/peersrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/routerrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/signrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/walletrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/watchtowerrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/wtclientrpc"
+	"github.com/lightningnetwork/lnd/lntest/wait"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+const (
+	DefaultTimeout = wait.DefaultTimeout
+)
+
+// HarnessRPC wraps all lnd's RPC clients into a single struct for easier
+// access.
+type HarnessRPC struct {
+	*testing.T
+
+	LN               lnrpc.LightningClient
+	WalletUnlocker   lnrpc.WalletUnlockerClient
+	Invoice          invoicesrpc.InvoicesClient
+	Signer           signrpc.SignerClient
+	Router           routerrpc.RouterClient
+	WalletKit        walletrpc.WalletKitClient
+	Watchtower       watchtowerrpc.WatchtowerClient
+	WatchtowerClient wtclientrpc.WatchtowerClientClient
+	State            lnrpc.StateClient
+	ChainClient      chainrpc.ChainNotifierClient
+	ChainKit         chainrpc.ChainKitClient
+	NeutrinoKit      neutrinorpc.NeutrinoKitClient
+	Peer             peersrpc.PeersClient
+
+	// Name is the HarnessNode's name.
+	Name string
+
+	// runCtx is a context with cancel method. It's used to signal when the
+	// node needs to quit, and used as the parent context when spawning
+	// children contexts for RPC requests.
+	runCtx context.Context
+	cancel context.CancelFunc
+}
+
+// NewHarnessRPC creates a new HarnessRPC with its own context inherted from
+// the pass context.
+func NewHarnessRPC(ctxt context.Context, t *testing.T, c *grpc.ClientConn,
+	name string) *HarnessRPC {
+
+	h := &HarnessRPC{
+		T:                t,
+		LN:               lnrpc.NewLightningClient(c),
+		Invoice:          invoicesrpc.NewInvoicesClient(c),
+		Router:           routerrpc.NewRouterClient(c),
+		WalletKit:        walletrpc.NewWalletKitClient(c),
+		WalletUnlocker:   lnrpc.NewWalletUnlockerClient(c),
+		Watchtower:       watchtowerrpc.NewWatchtowerClient(c),
+		WatchtowerClient: wtclientrpc.NewWatchtowerClientClient(c),
+		Signer:           signrpc.NewSignerClient(c),
+		State:            lnrpc.NewStateClient(c),
+		ChainClient:      chainrpc.NewChainNotifierClient(c),
+		ChainKit:         chainrpc.NewChainKitClient(c),
+		NeutrinoKit:      neutrinorpc.NewNeutrinoKitClient(c),
+		Peer:             peersrpc.NewPeersClient(c),
+		Name:             name,
+	}
+
+	// Inherit parent context.
+	h.runCtx, h.cancel = context.WithCancel(ctxt)
+	return h
+}
+
+// MakeOutpoint returns the outpoint of the channel's funding transaction.
+func (h *HarnessRPC) MakeOutpoint(cp *lnrpc.ChannelPoint) wire.OutPoint {
+	fundingTxID, err := lnrpc.GetChanPointFundingTxid(cp)
+	require.NoErrorf(h, err, "failed to make chanPoint", h.Name)
+
+	return wire.OutPoint{
+		Hash:  *fundingTxID,
+		Index: cp.OutputIndex,
+	}
+}
+
+// NoError is a helper method to format the error message used in calling RPCs.
+func (h *HarnessRPC) NoError(err error, operation string) {
+	require.NoErrorf(h, err, "%s: failed to call %s", h.Name, operation)
+}

lntest/rpc/invoices.go

@@ -0,0 +1,101 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/lightningnetwork/lnd/lnrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/invoicesrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/routerrpc"
+)
+
+// =====================
+// InvoiceClient related RPCs.
+// =====================
+
+// LookupInvoiceV2 queries the node's invoices using the invoice client's
+// LookupInvoiceV2.
+func (h *HarnessRPC) LookupInvoiceV2(
+	req *invoicesrpc.LookupInvoiceMsg) *lnrpc.Invoice {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Invoice.LookupInvoiceV2(ctxt, req)
+	h.NoError(err, "LookupInvoiceV2")
+
+	return resp
+}
+
+// AddHoldInvoice adds a hold invoice for the given node and asserts.
+func (h *HarnessRPC) AddHoldInvoice(
+	r *invoicesrpc.AddHoldInvoiceRequest) *invoicesrpc.AddHoldInvoiceResp {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	invoice, err := h.Invoice.AddHoldInvoice(ctxt, r)
+	h.NoError(err, "AddHoldInvoice")
+
+	return invoice
+}
+
+// SettleInvoice settles a given invoice and asserts.
+func (h *HarnessRPC) SettleInvoice(
+	preimage []byte) *invoicesrpc.SettleInvoiceResp {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &invoicesrpc.SettleInvoiceMsg{Preimage: preimage}
+	resp, err := h.Invoice.SettleInvoice(ctxt, req)
+	h.NoError(err, "SettleInvoice")
+
+	return resp
+}
+
+// CancelInvoice cancels a given invoice and asserts.
+func (h *HarnessRPC) CancelInvoice(
+	payHash []byte) *invoicesrpc.CancelInvoiceResp {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &invoicesrpc.CancelInvoiceMsg{PaymentHash: payHash}
+	resp, err := h.Invoice.CancelInvoice(ctxt, req)
+	h.NoError(err, "CancelInvoice")
+
+	return resp
+}
+
+type SingleInvoiceClient invoicesrpc.Invoices_SubscribeSingleInvoiceClient
+
+// SubscribeSingleInvoice creates a subscription client for given invoice and
+// asserts its creation.
+func (h *HarnessRPC) SubscribeSingleInvoice(rHash []byte) SingleInvoiceClient {
+	req := &invoicesrpc.SubscribeSingleInvoiceRequest{RHash: rHash}
+
+	// SubscribeSingleInvoice needs to have the context alive for the
+	// entire test case as the returned client will be used for send and
+	// receive events stream. Thus we use runCtx here instead of a timeout
+	// context.
+	client, err := h.Invoice.SubscribeSingleInvoice(h.runCtx, req)
+	h.NoError(err, "SubscribeSingleInvoice")
+
+	return client
+}
+
+type TrackPaymentClient routerrpc.Router_TrackPaymentV2Client
+
+// TrackPaymentV2 creates a subscription client for given invoice and
+// asserts its creation.
+func (h *HarnessRPC) TrackPaymentV2(payHash []byte) TrackPaymentClient {
+	req := &routerrpc.TrackPaymentRequest{PaymentHash: payHash}
+
+	// TrackPaymentV2 needs to have the context alive for the entire test
+	// case as the returned client will be used for send and receive events
+	// stream. Thus we use runCtx here instead of a timeout context.
+	client, err := h.Router.TrackPaymentV2(h.runCtx, req)
+	h.NoError(err, "TrackPaymentV2")
+
+	return client
+}

lntest/rpc/lnd.go

@@ -0,0 +1,684 @@
+package rpc
+
+import (
+	"context"
+	"strings"
+
+	"github.com/lightningnetwork/lnd/lnrpc"
+	"github.com/stretchr/testify/require"
+)
+
+// =====================
+// LightningClient related RPCs.
+// =====================
+
+// NewAddress makes a RPC call to NewAddress and asserts.
+func (h *HarnessRPC) NewAddress(
+	req *lnrpc.NewAddressRequest) *lnrpc.NewAddressResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.NewAddress(ctxt, req)
+	h.NoError(err, "NewAddress")
+
+	return resp
+}
+
+// WalletBalance makes a RPC call to WalletBalance and asserts.
+func (h *HarnessRPC) WalletBalance() *lnrpc.WalletBalanceResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &lnrpc.WalletBalanceRequest{}
+	resp, err := h.LN.WalletBalance(ctxt, req)
+	h.NoError(err, "WalletBalance")
+
+	return resp
+}
+
+// ListPeers makes a RPC call to the node's ListPeers and asserts.
+func (h *HarnessRPC) ListPeers() *lnrpc.ListPeersResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.ListPeers(ctxt, &lnrpc.ListPeersRequest{})
+	h.NoError(err, "ListPeers")
+
+	return resp
+}
+
+// DisconnectPeer calls the DisconnectPeer RPC on a given node with a specified
+// public key string and asserts there's no error.
+func (h *HarnessRPC) DisconnectPeer(
+	pubkey string) *lnrpc.DisconnectPeerResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &lnrpc.DisconnectPeerRequest{PubKey: pubkey}
+
+	resp, err := h.LN.DisconnectPeer(ctxt, req)
+	h.NoError(err, "DisconnectPeer")
+
+	return resp
+}
+
+// DeleteAllPayments makes a RPC call to the node's DeleteAllPayments and
+// asserts.
+func (h *HarnessRPC) DeleteAllPayments() {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &lnrpc.DeleteAllPaymentsRequest{}
+	_, err := h.LN.DeleteAllPayments(ctxt, req)
+	h.NoError(err, "DeleteAllPayments")
+}
+
+// GetInfo calls the GetInfo RPC on a given node and asserts there's no error.
+func (h *HarnessRPC) GetInfo() *lnrpc.GetInfoResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	info, err := h.LN.GetInfo(ctxt, &lnrpc.GetInfoRequest{})
+	h.NoError(err, "GetInfo")
+
+	return info
+}
+
+// ConnectPeer makes a RPC call to ConnectPeer and asserts there's no error.
+func (h *HarnessRPC) ConnectPeer(
+	req *lnrpc.ConnectPeerRequest) *lnrpc.ConnectPeerResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.ConnectPeer(ctxt, req)
+	h.NoError(err, "ConnectPeer")
+
+	return resp
+}
+
+// ConnectPeerAssertErr makes a RPC call to ConnectPeer and asserts an error
+// returned.
+func (h *HarnessRPC) ConnectPeerAssertErr(req *lnrpc.ConnectPeerRequest) error {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.LN.ConnectPeer(ctxt, req)
+	require.Error(h, err, "expected an error from ConnectPeer")
+
+	return err
+}
+
+// ListChannels list the channels for the given node and asserts it's
+// successful.
+func (h *HarnessRPC) ListChannels(
+	req *lnrpc.ListChannelsRequest) *lnrpc.ListChannelsResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.ListChannels(ctxt, req)
+	h.NoError(err, "ListChannels")
+
+	return resp
+}
+
+// PendingChannels makes a RPC request to PendingChannels and asserts there's
+// no error.
+func (h *HarnessRPC) PendingChannels() *lnrpc.PendingChannelsResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	pendingChansRequest := &lnrpc.PendingChannelsRequest{}
+	resp, err := h.LN.PendingChannels(ctxt, pendingChansRequest)
+
+	// TODO(yy): We may get a `unable to find arbitrator` error from the
+	// rpc point, due to a timing issue in rpcserver,
+	// 1. `r.server.chanStateDB.FetchClosedChannels` fetches
+	//    the pending force close channel.
+	// 2. `r.arbitratorPopulateForceCloseResp` relies on the
+	//    channel arbitrator to get the report, and,
+	// 3. the arbitrator may be deleted due to the force close
+	//    channel being resolved.
+	// Somewhere along the line is missing a lock to keep the data
+	// consistent.
+	//
+	// Return if there's no error.
+	if err == nil {
+		return resp
+	}
+
+	// Otherwise, give it a second shot if it's the arbitrator error.
+	if strings.Contains(err.Error(), "unable to find arbitrator") {
+		resp, err = h.LN.PendingChannels(ctxt, pendingChansRequest)
+	}
+
+	// It's very unlikely we'd get the arbitrator not found error again.
+	h.NoError(err, "PendingChannels")
+
+	return resp
+}
+
+// ClosedChannels makes a RPC call to node's ClosedChannels and asserts.
+func (h *HarnessRPC) ClosedChannels(
+	req *lnrpc.ClosedChannelsRequest) *lnrpc.ClosedChannelsResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.ClosedChannels(ctxt, req)
+	h.NoError(err, "ClosedChannels")
+
+	return resp
+}
+
+// ListPayments lists the node's payments and asserts.
+func (h *HarnessRPC) ListPayments(
+	req *lnrpc.ListPaymentsRequest) *lnrpc.ListPaymentsResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.ListPayments(ctxt, req)
+	h.NoError(err, "ListPayments")
+
+	return resp
+}
+
+// ListInvoices list the node's invoice using the request and asserts.
+func (h *HarnessRPC) ListInvoices(
+	req *lnrpc.ListInvoiceRequest) *lnrpc.ListInvoiceResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	if req == nil {
+		req = &lnrpc.ListInvoiceRequest{}
+	}
+
+	resp, err := h.LN.ListInvoices(ctxt, req)
+	h.NoError(err, "ListInvoice")
+
+	return resp
+}
+
+// DescribeGraph makes a RPC call to the node's DescribeGraph and asserts. It
+// takes a bool to indicate whether we want to include private edges or not.
+func (h *HarnessRPC) DescribeGraph(
+	req *lnrpc.ChannelGraphRequest) *lnrpc.ChannelGraph {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.DescribeGraph(ctxt, req)
+	h.NoError(err, "DescribeGraph")
+
+	return resp
+}
+
+// ChannelBalance gets the channel balance and asserts.
+func (h *HarnessRPC) ChannelBalance() *lnrpc.ChannelBalanceResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &lnrpc.ChannelBalanceRequest{}
+	resp, err := h.LN.ChannelBalance(ctxt, req)
+	h.NoError(err, "ChannelBalance")
+
+	return resp
+}
+
+type OpenChanClient lnrpc.Lightning_OpenChannelClient
+
+// OpenChannel makes a rpc call to LightningClient and returns the open channel
+// client.
+func (h *HarnessRPC) OpenChannel(req *lnrpc.OpenChannelRequest) OpenChanClient {
+	stream, err := h.LN.OpenChannel(h.runCtx, req)
+	h.NoError(err, "OpenChannel")
+
+	return stream
+}
+
+type CloseChanClient lnrpc.Lightning_CloseChannelClient
+
+// CloseChannel makes a rpc call to LightningClient and returns the close
+// channel client.
+func (h *HarnessRPC) CloseChannel(
+	req *lnrpc.CloseChannelRequest) CloseChanClient {
+
+	// Use runCtx here instead of a timeout context to keep the client
+	// alive for the entire test case.
+	stream, err := h.LN.CloseChannel(h.runCtx, req)
+	h.NoError(err, "CloseChannel")
+
+	return stream
+}
+
+// FundingStateStep makes a RPC call to FundingStateStep and asserts.
+func (h *HarnessRPC) FundingStateStep(
+	msg *lnrpc.FundingTransitionMsg) *lnrpc.FundingStateStepResp {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.FundingStateStep(ctxt, msg)
+	h.NoError(err, "FundingStateStep")
+
+	return resp
+}
+
+// FundingStateStepAssertErr makes a RPC call to FundingStateStep and asserts
+// there's an error.
+func (h *HarnessRPC) FundingStateStepAssertErr(m *lnrpc.FundingTransitionMsg) {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.LN.FundingStateStep(ctxt, m)
+	require.Error(h, err, "expected an error from FundingStateStep")
+}
+
+// AddInvoice adds a invoice for the given node and asserts.
+func (h *HarnessRPC) AddInvoice(req *lnrpc.Invoice) *lnrpc.AddInvoiceResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	invoice, err := h.LN.AddInvoice(ctxt, req)
+	h.NoError(err, "AddInvoice")
+
+	return invoice
+}
+
+// AbandonChannel makes a RPC call to AbandonChannel and asserts.
+func (h *HarnessRPC) AbandonChannel(
+	req *lnrpc.AbandonChannelRequest) *lnrpc.AbandonChannelResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.AbandonChannel(ctxt, req)
+	h.NoError(err, "AbandonChannel")
+
+	return resp
+}
+
+// ExportAllChanBackups makes a RPC call to the node's ExportAllChannelBackups
+// and asserts.
+func (h *HarnessRPC) ExportAllChanBackups() *lnrpc.ChanBackupSnapshot {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &lnrpc.ChanBackupExportRequest{}
+	chanBackup, err := h.LN.ExportAllChannelBackups(ctxt, req)
+	h.NoError(err, "ExportAllChannelBackups")
+
+	return chanBackup
+}
+
+// ExportChanBackup makes a RPC call to the node's ExportChannelBackup
+// and asserts.
+func (h *HarnessRPC) ExportChanBackup(
+	chanPoint *lnrpc.ChannelPoint) *lnrpc.ChannelBackup {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &lnrpc.ExportChannelBackupRequest{
+		ChanPoint: chanPoint,
+	}
+	chanBackup, err := h.LN.ExportChannelBackup(ctxt, req)
+	h.NoError(err, "ExportChannelBackup")
+
+	return chanBackup
+}
+
+// RestoreChanBackups makes a RPC call to the node's RestoreChannelBackups and
+// asserts.
+func (h *HarnessRPC) RestoreChanBackups(
+	req *lnrpc.RestoreChanBackupRequest) *lnrpc.RestoreBackupResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.RestoreChannelBackups(ctxt, req)
+	h.NoError(err, "RestoreChannelBackups")
+
+	return resp
+}
+
+type AcceptorClient lnrpc.Lightning_ChannelAcceptorClient
+
+// ChannelAcceptor makes a RPC call to the node's ChannelAcceptor and asserts.
+func (h *HarnessRPC) ChannelAcceptor() (AcceptorClient, context.CancelFunc) {
+	// Use runCtx here instead of a timeout context to keep the client
+	// alive for the entire test case.
+	ctxt, cancel := context.WithCancel(h.runCtx)
+	resp, err := h.LN.ChannelAcceptor(ctxt)
+	h.NoError(err, "ChannelAcceptor")
+
+	return resp, cancel
+}
+
+// SendCoins sends a given amount of money to the specified address from the
+// passed node.
+func (h *HarnessRPC) SendCoins(
+	req *lnrpc.SendCoinsRequest) *lnrpc.SendCoinsResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.SendCoins(ctxt, req)
+	h.NoError(err, "SendCoins")
+
+	return resp
+}
+
+// SendCoinsAssertErr sends a given amount of money to the specified address
+// from the passed node and asserts an error has returned.
+func (h *HarnessRPC) SendCoinsAssertErr(req *lnrpc.SendCoinsRequest) {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.LN.SendCoins(ctxt, req)
+	require.Error(h, err, "node %s didn't not return an error", h.Name)
+}
+
+// GetTransactions makes a RPC call to GetTransactions and asserts.
+func (h *HarnessRPC) GetTransactions(
+	req *lnrpc.GetTransactionsRequest) *lnrpc.TransactionDetails {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	if req == nil {
+		req = &lnrpc.GetTransactionsRequest{}
+	}
+
+	resp, err := h.LN.GetTransactions(ctxt, req)
+	h.NoError(err, "GetTransactions")
+
+	return resp
+}
+
+// SignMessage makes a RPC call to node's SignMessage and asserts.
+func (h *HarnessRPC) SignMessage(msg []byte) *lnrpc.SignMessageResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &lnrpc.SignMessageRequest{Msg: msg}
+	resp, err := h.LN.SignMessage(ctxt, req)
+	h.NoError(err, "SignMessage")
+
+	return resp
+}
+
+// VerifyMessage makes a RPC call to node's VerifyMessage and asserts.
+func (h *HarnessRPC) VerifyMessage(msg []byte,
+	sig string) *lnrpc.VerifyMessageResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &lnrpc.VerifyMessageRequest{Msg: msg, Signature: sig}
+	resp, err := h.LN.VerifyMessage(ctxt, req)
+	h.NoError(err, "VerifyMessage")
+
+	return resp
+}
+
+// GetRecoveryInfo uses the specified node to make a RPC call to
+// GetRecoveryInfo and asserts.
+func (h *HarnessRPC) GetRecoveryInfo(
+	req *lnrpc.GetRecoveryInfoRequest) *lnrpc.GetRecoveryInfoResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	if req == nil {
+		req = &lnrpc.GetRecoveryInfoRequest{}
+	}
+
+	resp, err := h.LN.GetRecoveryInfo(ctxt, req)
+	h.NoError(err, "GetRecoveryInfo")
+
+	return resp
+}
+
+// BatchOpenChannel makes a RPC call to BatchOpenChannel and asserts.
+func (h *HarnessRPC) BatchOpenChannel(
+	req *lnrpc.BatchOpenChannelRequest) *lnrpc.BatchOpenChannelResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.BatchOpenChannel(ctxt, req)
+	h.NoError(err, "BatchOpenChannel")
+
+	return resp
+}
+
+// BatchOpenChannelAssertErr makes a RPC call to BatchOpenChannel and asserts
+// there's an error returned.
+func (h *HarnessRPC) BatchOpenChannelAssertErr(
+	req *lnrpc.BatchOpenChannelRequest) error {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.LN.BatchOpenChannel(ctxt, req)
+	require.Error(h, err, "expecte batch open channel fail")
+
+	return err
+}
+
+// QueryRoutes makes a RPC call to QueryRoutes and asserts.
+func (h *HarnessRPC) QueryRoutes(
+	req *lnrpc.QueryRoutesRequest) *lnrpc.QueryRoutesResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	routes, err := h.LN.QueryRoutes(ctxt, req)
+	h.NoError(err, "QueryRoutes")
+
+	return routes
+}
+
+type SendToRouteClient lnrpc.Lightning_SendToRouteClient
+
+// SendToRoute makes a RPC call to SendToRoute and asserts.
+func (h *HarnessRPC) SendToRoute() SendToRouteClient {
+	// SendToRoute needs to have the context alive for the entire test case
+	// as the returned client will be used for send and receive payment
+	// stream. Thus we use runCtx here instead of a timeout context.
+	client, err := h.LN.SendToRoute(h.runCtx)
+	h.NoError(err, "SendToRoute")
+
+	return client
+}
+
+// SendToRouteSync makes a RPC call to SendToRouteSync and asserts.
+func (h *HarnessRPC) SendToRouteSync(
+	req *lnrpc.SendToRouteRequest) *lnrpc.SendResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.SendToRouteSync(ctxt, req)
+	h.NoError(err, "SendToRouteSync")
+
+	return resp
+}
+
+// UpdateChannelPolicy makes a RPC call to UpdateChannelPolicy and asserts.
+func (h *HarnessRPC) UpdateChannelPolicy(
+	req *lnrpc.PolicyUpdateRequest) *lnrpc.PolicyUpdateResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.UpdateChannelPolicy(ctxt, req)
+	h.NoError(err, "UpdateChannelPolicy")
+
+	return resp
+}
+
+type InvoiceUpdateClient lnrpc.Lightning_SubscribeInvoicesClient
+
+// SubscribeInvoices creates a subscription client for invoice events and
+// asserts its creation.
+//
+// NOTE: make sure to subscribe an invoice as early as possible as it takes
+// some time for the lnd to create the subscription client. If an invoice is
+// added right after the subscription, it may be missed. However, if AddIndex
+// or SettleIndex is used in the request, it will be fine as a backlog will
+// always be sent.
+func (h *HarnessRPC) SubscribeInvoices(
+	req *lnrpc.InvoiceSubscription) InvoiceUpdateClient {
+
+	// SubscribeInvoices needs to have the context alive for the
+	// entire test case as the returned client will be used for send and
+	// receive events stream. Thus we use runCtx here instead of a timeout
+	// context.
+	client, err := h.LN.SubscribeInvoices(h.runCtx, req)
+	h.NoError(err, "SubscribeInvoices")
+
+	return client
+}
+
+type BackupSubscriber lnrpc.Lightning_SubscribeChannelBackupsClient
+
+// SubscribeChannelBackups creates a client to listen to channel backup stream.
+func (h *HarnessRPC) SubscribeChannelBackups() BackupSubscriber {
+	// Use runCtx here instead of timeout context to keep the stream client
+	// alive.
+	backupStream, err := h.LN.SubscribeChannelBackups(
+		h.runCtx, &lnrpc.ChannelBackupSubscription{},
+	)
+	h.NoError(err, "SubscribeChannelBackups")
+
+	return backupStream
+}
+
+// VerifyChanBackup makes a RPC call to node's VerifyChanBackup and asserts.
+func (h *HarnessRPC) VerifyChanBackup(
+	ss *lnrpc.ChanBackupSnapshot) *lnrpc.VerifyChanBackupResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.VerifyChanBackup(ctxt, ss)
+	h.NoError(err, "VerifyChanBackup")
+
+	return resp
+}
+
+// LookupInvoice queries the node's invoices using the specified rHash.
+func (h *HarnessRPC) LookupInvoice(rHash []byte) *lnrpc.Invoice {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	payHash := &lnrpc.PaymentHash{RHash: rHash}
+	resp, err := h.LN.LookupInvoice(ctxt, payHash)
+	h.NoError(err, "LookupInvoice")
+
+	return resp
+}
+
+// DecodePayReq makes a RPC call to node's DecodePayReq and asserts.
+func (h *HarnessRPC) DecodePayReq(req string) *lnrpc.PayReq {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	payReq := &lnrpc.PayReqString{PayReq: req}
+	resp, err := h.LN.DecodePayReq(ctxt, payReq)
+	h.NoError(err, "DecodePayReq")
+
+	return resp
+}
+
+// ForwardingHistory makes a RPC call to the node's ForwardingHistory and
+// asserts.
+func (h *HarnessRPC) ForwardingHistory(
+	req *lnrpc.ForwardingHistoryRequest) *lnrpc.ForwardingHistoryResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	if req == nil {
+		req = &lnrpc.ForwardingHistoryRequest{}
+	}
+
+	resp, err := h.LN.ForwardingHistory(ctxt, req)
+	h.NoError(err, "ForwardingHistory")
+
+	return resp
+}
+
+type MiddlewareClient lnrpc.Lightning_RegisterRPCMiddlewareClient
+
+// RegisterRPCMiddleware makes a RPC call to the node's RegisterRPCMiddleware
+// and asserts. It also returns a cancel context which can cancel the context
+// used by the client.
+func (h *HarnessRPC) RegisterRPCMiddleware() (MiddlewareClient,
+	context.CancelFunc) {
+
+	ctxt, cancel := context.WithCancel(h.runCtx)
+
+	stream, err := h.LN.RegisterRPCMiddleware(ctxt)
+	h.NoError(err, "RegisterRPCMiddleware")
+
+	return stream, cancel
+}
+
+type ChannelEventsClient lnrpc.Lightning_SubscribeChannelEventsClient
+
+// SubscribeChannelEvents creates a subscription client for channel events and
+// asserts its creation.
+func (h *HarnessRPC) SubscribeChannelEvents() ChannelEventsClient {
+	req := &lnrpc.ChannelEventSubscription{}
+
+	// SubscribeChannelEvents needs to have the context alive for the
+	// entire test case as the returned client will be used for send and
+	// receive events stream. Thus we use runCtx here instead of a timeout
+	// context.
+	client, err := h.LN.SubscribeChannelEvents(h.runCtx, req)
+	h.NoError(err, "SubscribeChannelEvents")
+
+	return client
+}
+
+type CustomMessageClient lnrpc.Lightning_SubscribeCustomMessagesClient
+
+// SubscribeCustomMessages creates a subscription client for custom messages.
+func (h *HarnessRPC) SubscribeCustomMessages() (CustomMessageClient,
+	context.CancelFunc) {
+
+	ctxt, cancel := context.WithCancel(h.runCtx)
+
+	req := &lnrpc.SubscribeCustomMessagesRequest{}
+
+	// SubscribeCustomMessages needs to have the context alive for the
+	// entire test case as the returned client will be used for send and
+	// receive events stream. Thus we use runCtx here instead of a timeout
+	// context.
+	stream, err := h.LN.SubscribeCustomMessages(ctxt, req)
+	h.NoError(err, "SubscribeCustomMessages")
+
+	return stream, cancel
+}
+
+// SendCustomMessage makes a RPC call to the node's SendCustomMessage and
+// returns the response.
+func (h *HarnessRPC) SendCustomMessage(
+	req *lnrpc.SendCustomMessageRequest) *lnrpc.SendCustomMessageResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.LN.SendCustomMessage(ctxt, req)
+	h.NoError(err, "SendCustomMessage")
+
+	return resp
+}

lntest/rpc/neutrino_kit.go

@@ -0,0 +1,54 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/lightningnetwork/lnd/lnrpc/neutrinorpc"
+)
+
+// =====================
+// NeutrinoKitClient related RPCs.
+// =====================
+
+// Status makes an RPC call to neutrino kit client's Status and asserts.
+func (h *HarnessRPC) Status(
+	req *neutrinorpc.StatusRequest) *neutrinorpc.StatusResponse {
+
+	if req == nil {
+		req = &neutrinorpc.StatusRequest{}
+	}
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.NeutrinoKit.Status(ctxt, req)
+	h.NoError(err, "Status")
+
+	return resp
+}
+
+// GetCFilter makes an RPC call to neutrino kit client's GetCFilter and asserts.
+func (h *HarnessRPC) GetCFilter(
+	req *neutrinorpc.GetCFilterRequest) *neutrinorpc.GetCFilterResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.NeutrinoKit.GetCFilter(ctxt, req)
+	h.NoError(err, "GetCFilter")
+
+	return resp
+}
+
+// AddPeer makes an RPC call to neutrino kit client's AddPeer and asserts.
+func (h *HarnessRPC) AddPeer(
+	req *neutrinorpc.AddPeerRequest) *neutrinorpc.AddPeerResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.NeutrinoKit.AddPeer(ctxt, req)
+	h.NoError(err, "AddPeer")
+
+	return resp
+}

lntest/rpc/peers.go

@@ -0,0 +1,39 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/lightningnetwork/lnd/lnrpc/peersrpc"
+	"github.com/stretchr/testify/require"
+)
+
+// =====================
+// PeerClient related RPCs.
+// =====================
+
+type (
+	AnnReq  *peersrpc.NodeAnnouncementUpdateRequest
+	AnnResp *peersrpc.NodeAnnouncementUpdateResponse
+)
+
+// UpdateNodeAnnouncement makes an UpdateNodeAnnouncement RPC call the peersrpc
+// client and asserts.
+func (h *HarnessRPC) UpdateNodeAnnouncement(req AnnReq) AnnResp {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Peer.UpdateNodeAnnouncement(ctxt, req)
+	h.NoError(err, "UpdateNodeAnnouncement")
+
+	return resp
+}
+
+// UpdateNodeAnnouncementErr makes an UpdateNodeAnnouncement RPC call the
+// peersrpc client and asserts an error is returned.
+func (h *HarnessRPC) UpdateNodeAnnouncementErr(req AnnReq) {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.Peer.UpdateNodeAnnouncement(ctxt, req)
+	require.Error(h, err, "expect an error from update announcement")
+}

lntest/rpc/router.go

@@ -0,0 +1,206 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/lightningnetwork/lnd/lnrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/routerrpc"
+	"github.com/stretchr/testify/require"
+)
+
+// =====================
+// RouterClient related RPCs.
+// =====================
+
+// UpdateChanStatus makes a UpdateChanStatus RPC call to node's RouterClient
+// and asserts.
+//
+//nolint:lll
+func (h *HarnessRPC) UpdateChanStatus(
+	req *routerrpc.UpdateChanStatusRequest) *routerrpc.UpdateChanStatusResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Router.UpdateChanStatus(ctxt, req)
+	h.NoError(err, "UpdateChanStatus")
+
+	return resp
+}
+
+type PaymentClient routerrpc.Router_SendPaymentV2Client
+
+// SendPayment sends a payment using the given node and payment request. It
+// also asserts the payment being sent successfully.
+func (h *HarnessRPC) SendPayment(
+	req *routerrpc.SendPaymentRequest) PaymentClient {
+
+	// SendPayment needs to have the context alive for the entire test case
+	// as the router relies on the context to propagate HTLCs. Thus we use
+	// runCtx here instead of a timeout context.
+	stream, err := h.Router.SendPaymentV2(h.runCtx, req)
+	h.NoError(err, "SendPaymentV2")
+
+	return stream
+}
+
+type HtlcEventsClient routerrpc.Router_SubscribeHtlcEventsClient
+
+// SubscribeHtlcEvents makes a subscription to the HTLC events and returns a
+// htlc event client.
+func (h *HarnessRPC) SubscribeHtlcEvents() HtlcEventsClient {
+	// Use runCtx here to keep the client alive for the scope of the test.
+	client, err := h.Router.SubscribeHtlcEvents(
+		h.runCtx, &routerrpc.SubscribeHtlcEventsRequest{},
+	)
+	h.NoError(err, "SubscribeHtlcEvents")
+
+	return client
+}
+
+// GetMissionControlConfig makes a RPC call to the node's
+// GetMissionControlConfig and asserts.
+//
+//nolint:lll
+func (h *HarnessRPC) GetMissionControlConfig() *routerrpc.GetMissionControlConfigResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &routerrpc.GetMissionControlConfigRequest{}
+	resp, err := h.Router.GetMissionControlConfig(ctxt, req)
+	h.NoError(err, "GetMissionControlConfig")
+
+	return resp
+}
+
+// SetMissionControlConfig makes a RPC call to the node's
+// SetMissionControlConfig and asserts.
+func (h *HarnessRPC) SetMissionControlConfig(
+	config *routerrpc.MissionControlConfig) {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &routerrpc.SetMissionControlConfigRequest{Config: config}
+	_, err := h.Router.SetMissionControlConfig(ctxt, req)
+	h.NoError(err, "SetMissionControlConfig")
+}
+
+// SetMissionControlConfigAssertErr makes a RPC call to the node's
+// SetMissionControlConfig and asserts that we error.
+func (h *HarnessRPC) SetMissionControlConfigAssertErr(
+	config *routerrpc.MissionControlConfig) {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &routerrpc.SetMissionControlConfigRequest{Config: config}
+	_, err := h.Router.SetMissionControlConfig(ctxt, req)
+	require.Error(h, err, "expect an error from setting import mission "+
+		"control")
+}
+
+// ResetMissionControl makes a RPC call to the node's ResetMissionControl and
+// asserts.
+func (h *HarnessRPC) ResetMissionControl() {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &routerrpc.ResetMissionControlRequest{}
+	_, err := h.Router.ResetMissionControl(ctxt, req)
+	h.NoError(err, "ResetMissionControl")
+}
+
+// SendToRouteV2 makes a RPC call to SendToRouteV2 and asserts.
+func (h *HarnessRPC) SendToRouteV2(
+	req *routerrpc.SendToRouteRequest) *lnrpc.HTLCAttempt {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Router.SendToRouteV2(ctxt, req)
+	h.NoError(err, "SendToRouteV2")
+
+	return resp
+}
+
+// QueryProbability makes a RPC call to the node's QueryProbability and
+// asserts.
+//
+//nolint:lll
+func (h *HarnessRPC) QueryProbability(
+	req *routerrpc.QueryProbabilityRequest) *routerrpc.QueryProbabilityResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Router.QueryProbability(ctxt, req)
+	h.NoError(err, "QueryProbability")
+
+	return resp
+}
+
+// XImportMissionControl makes a RPC call to the node's XImportMissionControl
+// and asserts.
+func (h *HarnessRPC) XImportMissionControl(
+	req *routerrpc.XImportMissionControlRequest) {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.Router.XImportMissionControl(ctxt, req)
+	h.NoError(err, "XImportMissionControl")
+}
+
+// XImportMissionControlAssertErr makes a RPC call to the node's
+// XImportMissionControl
+// and asserts an error occurred.
+func (h *HarnessRPC) XImportMissionControlAssertErr(
+	req *routerrpc.XImportMissionControlRequest) {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.Router.XImportMissionControl(ctxt, req)
+	require.Error(h, err, "expect an error from x import mission control")
+}
+
+// BuildRoute makes a RPC call to the node's RouterClient and asserts.
+func (h *HarnessRPC) BuildRoute(
+	req *routerrpc.BuildRouteRequest) *routerrpc.BuildRouteResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Router.BuildRoute(ctxt, req)
+	h.NoError(err, "BuildRoute")
+
+	return resp
+}
+
+type InterceptorClient routerrpc.Router_HtlcInterceptorClient
+
+// HtlcInterceptor makes a RPC call to the node's RouterClient and asserts.
+func (h *HarnessRPC) HtlcInterceptor() (InterceptorClient, context.CancelFunc) {
+	// HtlcInterceptor needs to have the context alive for the entire test
+	// case as the returned client will be used for send and receive events
+	// stream. Thus we use cancel context here instead of a timeout
+	// context.
+	ctxt, cancel := context.WithCancel(h.runCtx)
+	resp, err := h.Router.HtlcInterceptor(ctxt)
+	h.NoError(err, "HtlcInterceptor")
+
+	return resp, cancel
+}
+
+type TrackPaymentsClient routerrpc.Router_TrackPaymentsClient
+
+// TrackPayments makes a RPC call to the node's RouterClient and asserts.
+func (h *HarnessRPC) TrackPayments(
+	req *routerrpc.TrackPaymentsRequest) TrackPaymentsClient {
+
+	resp, err := h.Router.TrackPayments(h.runCtx, req)
+	h.NoError(err, "TrackPayments")
+
+	return resp
+}

lntest/rpc/signer.go

@@ -0,0 +1,225 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/lightningnetwork/lnd/lnrpc/signrpc"
+	"github.com/stretchr/testify/require"
+)
+
+// =====================
+// Signer related RPCs.
+// =====================
+
+// DeriveSharedKey makes a RPC call to the node's SignerClient and asserts.
+func (h *HarnessRPC) DeriveSharedKey(
+	req *signrpc.SharedKeyRequest) *signrpc.SharedKeyResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.DeriveSharedKey(ctxt, req)
+	h.NoError(err, "DeriveSharedKey")
+
+	return resp
+}
+
+// DeriveSharedKeyErr makes a RPC call to the node's SignerClient and asserts
+// there is an error.
+func (h *HarnessRPC) DeriveSharedKeyErr(req *signrpc.SharedKeyRequest) error {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.Signer.DeriveSharedKey(ctxt, req)
+	require.Error(h, err, "expected error from calling DeriveSharedKey")
+
+	return err
+}
+
+// SignOutputRaw makes a RPC call to the node's SignerClient and asserts.
+func (h *HarnessRPC) SignOutputRaw(req *signrpc.SignReq) *signrpc.SignResp {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.SignOutputRaw(ctxt, req)
+	h.NoError(err, "SignOutputRaw")
+
+	return resp
+}
+
+// SignOutputRawErr makes a RPC call to the node's SignerClient and asserts an
+// error is returned.
+func (h *HarnessRPC) SignOutputRawErr(req *signrpc.SignReq) error {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.Signer.SignOutputRaw(ctxt, req)
+	require.Error(h, err, "expect to fail to sign raw output")
+
+	return err
+}
+
+// MuSig2CreateSession makes a RPC call to the node's SignerClient and asserts.
+func (h *HarnessRPC) MuSig2CreateSession(
+	req *signrpc.MuSig2SessionRequest) *signrpc.MuSig2SessionResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.MuSig2CreateSession(ctxt, req)
+	h.NoError(err, "MuSig2CreateSession")
+
+	return resp
+}
+
+// MuSig2CreateSessionErr makes an RPC call to the node's SignerClient and
+// asserts an error is returned.
+func (h *HarnessRPC) MuSig2CreateSessionErr(
+	req *signrpc.MuSig2SessionRequest) error {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.Signer.MuSig2CreateSession(ctxt, req)
+	require.Error(h, err, "expected error from calling MuSig2CreateSession")
+
+	return err
+}
+
+// MuSig2CombineKeys makes a RPC call to the node's SignerClient and asserts.
+//
+//nolint:lll
+func (h *HarnessRPC) MuSig2CombineKeys(
+	req *signrpc.MuSig2CombineKeysRequest) *signrpc.MuSig2CombineKeysResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.MuSig2CombineKeys(ctxt, req)
+	h.NoError(err, "MuSig2CombineKeys")
+
+	return resp
+}
+
+// MuSig2CombineKeysErr makes an RPC call to the node's SignerClient and
+// asserts an error is returned.
+func (h *HarnessRPC) MuSig2CombineKeysErr(
+	req *signrpc.MuSig2CombineKeysRequest) error {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.Signer.MuSig2CombineKeys(ctxt, req)
+	require.Error(h, err, "expected error from calling MuSig2CombineKeys")
+
+	return err
+}
+
+// MuSig2RegisterNonces makes a RPC call to the node's SignerClient and asserts.
+//
+//nolint:lll
+func (h *HarnessRPC) MuSig2RegisterNonces(
+	req *signrpc.MuSig2RegisterNoncesRequest) *signrpc.MuSig2RegisterNoncesResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.MuSig2RegisterNonces(ctxt, req)
+	h.NoError(err, "MuSig2RegisterNonces")
+
+	return resp
+}
+
+// MuSig2Sign makes a RPC call to the node's SignerClient and asserts.
+func (h *HarnessRPC) MuSig2Sign(
+	req *signrpc.MuSig2SignRequest) *signrpc.MuSig2SignResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.MuSig2Sign(ctxt, req)
+	h.NoError(err, "MuSig2Sign")
+
+	return resp
+}
+
+// MuSig2SignErr makes a RPC call to the node's SignerClient and asserts an
+// error is returned.
+func (h *HarnessRPC) MuSig2SignErr(req *signrpc.MuSig2SignRequest) error {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.Signer.MuSig2Sign(ctxt, req)
+	require.Error(h, err, "expect an error")
+
+	return err
+}
+
+// MuSig2CombineSig makes a RPC call to the node's SignerClient and asserts.
+func (h *HarnessRPC) MuSig2CombineSig(
+	r *signrpc.MuSig2CombineSigRequest) *signrpc.MuSig2CombineSigResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.MuSig2CombineSig(ctxt, r)
+	h.NoError(err, "MuSig2CombineSig")
+
+	return resp
+}
+
+// MuSig2Cleanup makes a RPC call to the node's SignerClient and asserts.
+func (h *HarnessRPC) MuSig2Cleanup(
+	req *signrpc.MuSig2CleanupRequest) *signrpc.MuSig2CleanupResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.MuSig2Cleanup(ctxt, req)
+	h.NoError(err, "MuSig2Cleanup")
+
+	return resp
+}
+
+// SignMessageSigner makes a RPC call to the node's SignerClient and asserts.
+//
+// NOTE: there's already `SignMessage` in `h.LN`.
+func (h *HarnessRPC) SignMessageSigner(
+	req *signrpc.SignMessageReq) *signrpc.SignMessageResp {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.SignMessage(ctxt, req)
+	h.NoError(err, "SignMessage")
+
+	return resp
+}
+
+// VerifyMessageSigner makes a RPC call to the node's SignerClient and asserts.
+//
+// NOTE: there's already `VerifyMessageSigner` in `h.LN`.
+func (h *HarnessRPC) VerifyMessageSigner(
+	req *signrpc.VerifyMessageReq) *signrpc.VerifyMessageResp {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.VerifyMessage(ctxt, req)
+	h.NoError(err, "VerifyMessage")
+
+	return resp
+}
+
+// ComputeInputScript makes a RPC call to the node's SignerClient and asserts.
+func (h *HarnessRPC) ComputeInputScript(
+	req *signrpc.SignReq) *signrpc.InputScriptResp {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.Signer.ComputeInputScript(ctxt, req)
+	h.NoError(err, "ComputeInputScript")
+
+	return resp
+}

lntest/rpc/state.go

@@ -0,0 +1,19 @@
+package rpc
+
+import (
+	"github.com/lightningnetwork/lnd/lnrpc"
+)
+
+// =====================
+// StateClient related RPCs.
+// =====================
+
+// SubscribeState makes a rpc call to StateClient and asserts there's no error.
+func (h *HarnessRPC) SubscribeState() lnrpc.State_SubscribeStateClient {
+	client, err := h.State.SubscribeState(
+		h.runCtx, &lnrpc.SubscribeStateRequest{},
+	)
+	h.NoError(err, "SubscribeState")
+
+	return client
+}

lntest/rpc/wallet_kit.go

@@ -0,0 +1,279 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/lightningnetwork/lnd/lnrpc/signrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/walletrpc"
+	"github.com/stretchr/testify/require"
+)
+
+// =====================
+// WalletKitClient related RPCs.
+// =====================
+
+type (
+	SignReq    *walletrpc.SignMessageWithAddrResponse
+	VerifyResp *walletrpc.VerifyMessageWithAddrResponse
+)
+
+// FinalizePsbt makes a RPC call to node's ListUnspent and asserts.
+func (h *HarnessRPC) ListUnspent(
+	req *walletrpc.ListUnspentRequest) *walletrpc.ListUnspentResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.ListUnspent(ctxt, req)
+	h.NoError(err, "ListUnspent")
+
+	return resp
+}
+
+// DeriveKey makes a RPC call to the DeriveKey and asserts.
+func (h *HarnessRPC) DeriveKey(kl *signrpc.KeyLocator) *signrpc.KeyDescriptor {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	key, err := h.WalletKit.DeriveKey(ctxt, kl)
+	h.NoError(err, "DeriveKey")
+
+	return key
+}
+
+// SendOutputs makes a RPC call to the node's WalletKitClient and asserts.
+func (h *HarnessRPC) SendOutputs(
+	req *walletrpc.SendOutputsRequest) *walletrpc.SendOutputsResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.SendOutputs(ctxt, req)
+	h.NoError(err, "SendOutputs")
+
+	return resp
+}
+
+// FundPsbt makes a RPC call to node's FundPsbt and asserts.
+func (h *HarnessRPC) FundPsbt(
+	req *walletrpc.FundPsbtRequest) *walletrpc.FundPsbtResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.FundPsbt(ctxt, req)
+	h.NoError(err, "FundPsbt")
+
+	return resp
+}
+
+// FinalizePsbt makes a RPC call to node's FinalizePsbt and asserts.
+func (h *HarnessRPC) FinalizePsbt(
+	req *walletrpc.FinalizePsbtRequest) *walletrpc.FinalizePsbtResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.FinalizePsbt(ctxt, req)
+	h.NoError(err, "FinalizePsbt")
+
+	return resp
+}
+
+// LabelTransactionAssertErr makes a RPC call to the node's LabelTransaction
+// and asserts an error is returned. It then returns the error.
+func (h *HarnessRPC) LabelTransactionAssertErr(
+	req *walletrpc.LabelTransactionRequest) error {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.WalletKit.LabelTransaction(ctxt, req)
+	require.Error(h, err, "expected error returned")
+
+	return err
+}
+
+// LabelTransaction makes a RPC call to the node's LabelTransaction
+// and asserts no error is returned.
+func (h *HarnessRPC) LabelTransaction(req *walletrpc.LabelTransactionRequest) {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	_, err := h.WalletKit.LabelTransaction(ctxt, req)
+	h.NoError(err, "LabelTransaction")
+}
+
+// DeriveNextKey makes a RPC call to the DeriveNextKey and asserts.
+func (h *HarnessRPC) DeriveNextKey(
+	req *walletrpc.KeyReq) *signrpc.KeyDescriptor {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	key, err := h.WalletKit.DeriveNextKey(ctxt, req)
+	h.NoError(err, "DeriveNextKey")
+
+	return key
+}
+
+// ListAddresses makes a RPC call to the ListAddresses and asserts.
+func (h *HarnessRPC) ListAddresses(
+	req *walletrpc.ListAddressesRequest) *walletrpc.ListAddressesResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	key, err := h.WalletKit.ListAddresses(ctxt, req)
+	h.NoError(err, "ListAddresses")
+
+	return key
+}
+
+// SignMessageWithAddr makes a RPC call to the SignMessageWithAddr and asserts.
+func (h *HarnessRPC) SignMessageWithAddr(
+	req *walletrpc.SignMessageWithAddrRequest) SignReq {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	key, err := h.WalletKit.SignMessageWithAddr(ctxt, req)
+	h.NoError(err, "SignMessageWithAddr")
+
+	return key
+}
+
+// VerifyMessageWithAddr makes a RPC call to
+// the VerifyMessageWithAddr and asserts.
+func (h *HarnessRPC) VerifyMessageWithAddr(
+	req *walletrpc.VerifyMessageWithAddrRequest) VerifyResp {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	key, err := h.WalletKit.VerifyMessageWithAddr(ctxt, req)
+	h.NoError(err, "VerifyMessageWithAddr")
+
+	return key
+}
+
+// ListSweeps makes a ListSweeps RPC call to the node's WalletKit client.
+func (h *HarnessRPC) ListSweeps(verbose bool) *walletrpc.ListSweepsResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &walletrpc.ListSweepsRequest{Verbose: verbose}
+	resp, err := h.WalletKit.ListSweeps(ctxt, req)
+	h.NoError(err, "ListSweeps")
+
+	return resp
+}
+
+// PendingSweeps makes a RPC call to the node's WalletKitClient and asserts.
+func (h *HarnessRPC) PendingSweeps() *walletrpc.PendingSweepsResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &walletrpc.PendingSweepsRequest{}
+
+	resp, err := h.WalletKit.PendingSweeps(ctxt, req)
+	h.NoError(err, "PendingSweeps")
+
+	return resp
+}
+
+// PublishTransaction makes an RPC call to the node's WalletKitClient and
+// asserts.
+func (h *HarnessRPC) PublishTransaction(
+	req *walletrpc.Transaction) *walletrpc.PublishResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.PublishTransaction(ctxt, req)
+	h.NoError(err, "PublishTransaction")
+
+	return resp
+}
+
+// BumpFee makes a RPC call to the node's WalletKitClient and asserts.
+func (h *HarnessRPC) BumpFee(
+	req *walletrpc.BumpFeeRequest) *walletrpc.BumpFeeResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.BumpFee(ctxt, req)
+	h.NoError(err, "BumpFee")
+
+	return resp
+}
+
+// ListAccounts makes a RPC call to the node's WalletKitClient and asserts.
+func (h *HarnessRPC) ListAccounts(
+	req *walletrpc.ListAccountsRequest) *walletrpc.ListAccountsResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.ListAccounts(ctxt, req)
+	h.NoError(err, "ListAccounts")
+
+	return resp
+}
+
+// ImportAccount makes a RPC call to the node's WalletKitClient and asserts.
+func (h *HarnessRPC) ImportAccount(
+	req *walletrpc.ImportAccountRequest) *walletrpc.ImportAccountResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.ImportAccount(ctxt, req)
+	h.NoError(err, "ImportAccount")
+
+	return resp
+}
+
+// ImportPublicKey makes a RPC call to the node's WalletKitClient and asserts.
+//
+//nolint:lll
+func (h *HarnessRPC) ImportPublicKey(
+	req *walletrpc.ImportPublicKeyRequest) *walletrpc.ImportPublicKeyResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.ImportPublicKey(ctxt, req)
+	h.NoError(err, "ImportPublicKey")
+
+	return resp
+}
+
+// SignPsbt makes a RPC call to the node's WalletKitClient and asserts.
+func (h *HarnessRPC) SignPsbt(
+	req *walletrpc.SignPsbtRequest) *walletrpc.SignPsbtResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.SignPsbt(ctxt, req)
+	h.NoError(err, "SignPsbt")
+
+	return resp
+}
+
+// ImportTapscript makes a RPC call to the node's WalletKitClient and asserts.
+//
+//nolint:lll
+func (h *HarnessRPC) ImportTapscript(
+	req *walletrpc.ImportTapscriptRequest) *walletrpc.ImportTapscriptResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletKit.ImportTapscript(ctxt, req)
+	h.NoError(err, "ImportTapscript")
+
+	return resp
+}

lntest/rpc/wallet_unlocker.go

@@ -0,0 +1,64 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/lightningnetwork/lnd/lnrpc"
+)
+
+// =====================
+// WalletUnlockerClient related RPCs.
+// =====================
+
+// UnlockWallet makes a RPC request to WalletUnlocker and asserts there's no
+// error.
+func (h *HarnessRPC) UnlockWallet(
+	req *lnrpc.UnlockWalletRequest) *lnrpc.UnlockWalletResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletUnlocker.UnlockWallet(ctxt, req)
+	h.NoError(err, "UnlockWallet")
+
+	return resp
+}
+
+// InitWallet makes a RPC request to WalletUnlocker and asserts there's no
+// error.
+func (h *HarnessRPC) InitWallet(
+	req *lnrpc.InitWalletRequest) *lnrpc.InitWalletResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletUnlocker.InitWallet(ctxt, req)
+	h.NoError(err, "InitWallet")
+
+	return resp
+}
+
+// GenSeed makes a RPC request to WalletUnlocker and asserts there's no error.
+func (h *HarnessRPC) GenSeed(req *lnrpc.GenSeedRequest) *lnrpc.GenSeedResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletUnlocker.GenSeed(ctxt, req)
+	h.NoError(err, "GenSeed")
+
+	return resp
+}
+
+// ChangePassword makes a RPC request to WalletUnlocker and asserts there's no
+// error.
+func (h *HarnessRPC) ChangePassword(
+	req *lnrpc.ChangePasswordRequest) *lnrpc.ChangePasswordResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WalletUnlocker.ChangePassword(ctxt, req)
+	h.NoError(err, "ChangePassword")
+
+	return resp
+}

lntest/rpc/watchtower.go

@@ -0,0 +1,52 @@
+package rpc
+
+import (
+	"context"
+
+	"github.com/lightningnetwork/lnd/lnrpc/watchtowerrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/wtclientrpc"
+)
+
+// =====================
+// WatchtowerClient and WatchtowerClientClient related RPCs.
+// =====================
+
+// GetInfoWatchtower makes a RPC call to the watchtower of the given node and
+// asserts.
+func (h *HarnessRPC) GetInfoWatchtower() *watchtowerrpc.GetInfoResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &watchtowerrpc.GetInfoRequest{}
+	info, err := h.Watchtower.GetInfo(ctxt, req)
+	h.NoError(err, "GetInfo from Watchtower")
+
+	return info
+}
+
+// AddTower makes a RPC call to the WatchtowerClient of the given node and
+// asserts.
+func (h *HarnessRPC) AddTower(
+	req *wtclientrpc.AddTowerRequest) *wtclientrpc.AddTowerResponse {
+
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	resp, err := h.WatchtowerClient.AddTower(ctxt, req)
+	h.NoError(err, "AddTower")
+
+	return resp
+}
+
+// WatchtowerStats makes a RPC call to the WatchtowerClient of the given node
+// and asserts.
+func (h *HarnessRPC) WatchtowerStats() *wtclientrpc.StatsResponse {
+	ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout)
+	defer cancel()
+
+	req := &wtclientrpc.StatsRequest{}
+	resp, err := h.WatchtowerClient.Stats(ctxt, req)
+	h.NoError(err, "Stats from Watchtower")
+
+	return resp
+}

lntest/utils.go

@@ -0,0 +1,200 @@
+package lntest
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/btcsuite/btcd/btcutil"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/lightningnetwork/lnd/input"
+	"github.com/lightningnetwork/lnd/lnrpc"
+	"github.com/lightningnetwork/lnd/lntest/wait"
+	"github.com/lightningnetwork/lnd/lnwallet"
+	"github.com/lightningnetwork/lnd/lnwallet/chainfee"
+	"github.com/lightningnetwork/lnd/lnwire"
+)
+
+const (
+	// NeutrinoBackendName is the name of the neutrino backend.
+	NeutrinoBackendName = "neutrino"
+
+	DefaultTimeout = wait.DefaultTimeout
+
+	// noFeeLimitMsat is used to specify we will put no requirements on fee
+	// charged when choosing a route path.
+	noFeeLimitMsat = math.MaxInt64
+
+	// defaultPaymentTimeout specifies the default timeout in seconds when
+	// sending a payment.
+	defaultPaymentTimeout = 60
+)
+
+// CopyFile copies the file src to dest.
+func CopyFile(dest, src string) error {
+	s, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	d, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(d, s); err != nil {
+		d.Close()
+		return err
+	}
+
+	return d.Close()
+}
+
+// errNumNotMatched is a helper method to return a nicely formatted error.
+func errNumNotMatched(name string, subject string,
+	want, got, total, old int) error {
+
+	return fmt.Errorf("%s: assert %s failed: want %d, got: %d, total: "+
+		"%d, previously had: %d", name, subject, want, got, total, old)
+}
+
+// parseDerivationPath parses a path in the form of m/x'/y'/z'/a/b into a slice
+// of [x, y, z, a, b], meaning that the apostrophe is ignored and 2^31 is _not_
+// added to the numbers.
+func ParseDerivationPath(path string) ([]uint32, error) {
+	path = strings.TrimSpace(path)
+	if len(path) == 0 {
+		return nil, fmt.Errorf("path cannot be empty")
+	}
+	if !strings.HasPrefix(path, "m/") {
+		return nil, fmt.Errorf("path must start with m/")
+	}
+
+	// Just the root key, no path was provided. This is valid but not useful
+	// in most cases.
+	rest := strings.ReplaceAll(path, "m/", "")
+	if rest == "" {
+		return []uint32{}, nil
+	}
+
+	parts := strings.Split(rest, "/")
+	indices := make([]uint32, len(parts))
+	for i := 0; i < len(parts); i++ {
+		part := parts[i]
+		if strings.Contains(parts[i], "'") {
+			part = strings.TrimRight(parts[i], "'")
+		}
+		parsed, err := strconv.ParseInt(part, 10, 32)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse part \"%s\": "+
+				"%v", part, err)
+		}
+		indices[i] = uint32(parsed)
+	}
+
+	return indices, nil
+}
+
+// ChanPointFromPendingUpdate constructs a channel point from a lnrpc pending
+// update.
+func ChanPointFromPendingUpdate(pu *lnrpc.PendingUpdate) *lnrpc.ChannelPoint {
+	chanPoint := &lnrpc.ChannelPoint{
+		FundingTxid: &lnrpc.ChannelPoint_FundingTxidBytes{
+			FundingTxidBytes: pu.Txid,
+		},
+		OutputIndex: pu.OutputIndex,
+	}
+
+	return chanPoint
+}
+
+// channelPointStr returns the string representation of the channel's
+// funding transaction.
+func channelPointStr(chanPoint *lnrpc.ChannelPoint) string {
+	fundingTxID, err := lnrpc.GetChanPointFundingTxid(chanPoint)
+	if err != nil {
+		return ""
+	}
+	cp := wire.OutPoint{
+		Hash:  *fundingTxID,
+		Index: chanPoint.OutputIndex,
+	}
+
+	return cp.String()
+}
+
+// CommitTypeHasAnchors returns whether commitType uses anchor outputs.
+func CommitTypeHasAnchors(commitType lnrpc.CommitmentType) bool {
+	switch commitType {
+	case lnrpc.CommitmentType_ANCHORS,
+		lnrpc.CommitmentType_SCRIPT_ENFORCED_LEASE:
+		return true
+	default:
+		return false
+	}
+}
+
+// NodeArgsForCommitType returns the command line flag to supply to enable this
+// commitment type.
+func NodeArgsForCommitType(commitType lnrpc.CommitmentType) []string {
+	switch commitType {
+	case lnrpc.CommitmentType_LEGACY:
+		return []string{"--protocol.legacy.committweak"}
+	case lnrpc.CommitmentType_STATIC_REMOTE_KEY:
+		return []string{}
+	case lnrpc.CommitmentType_ANCHORS:
+		return []string{"--protocol.anchors"}
+	case lnrpc.CommitmentType_SCRIPT_ENFORCED_LEASE:
+		return []string{
+			"--protocol.anchors",
+			"--protocol.script-enforced-lease",
+		}
+	}
+
+	return nil
+}
+
+// CalcStaticFee calculates appropriate fees for commitment transactions. This
+// function provides a simple way to allow test balance assertions to take fee
+// calculations into account.
+func CalcStaticFee(c lnrpc.CommitmentType, numHTLCs int) btcutil.Amount {
+	const (
+		htlcWeight         = input.HTLCWeight
+		anchorSize         = 330
+		defaultSatPerVByte = lnwallet.DefaultAnchorsCommitMaxFeeRateSatPerVByte
+	)
+
+	var (
+		anchors      = btcutil.Amount(0)
+		commitWeight = input.CommitWeight
+		feePerKw     = chainfee.SatPerKWeight(DefaultFeeRateSatPerKw)
+	)
+
+	// The anchor commitment type is slightly heavier, and we must also add
+	// the value of the two anchors to the resulting fee the initiator
+	// pays. In addition the fee rate is capped at 10 sat/vbyte for anchor
+	// channels.
+	if CommitTypeHasAnchors(c) {
+		feePerKw = chainfee.SatPerKVByte(
+			defaultSatPerVByte * 1000,
+		).FeePerKWeight()
+		commitWeight = input.AnchorCommitWeight
+		anchors = 2 * anchorSize
+	}
+
+	return feePerKw.FeeForWeight(int64(commitWeight+htlcWeight*numHTLCs)) +
+		anchors
+}
+
+// CalculateMaxHtlc re-implements the RequiredRemoteChannelReserve of the
+// funding manager's config, which corresponds to the maximum MaxHTLC value we
+// allow users to set when updating a channel policy.
+func CalculateMaxHtlc(chanCap btcutil.Amount) uint64 {
+	reserve := lnwire.NewMSatFromSatoshis(chanCap / 100)
+	max := lnwire.NewMSatFromSatoshis(chanCap) - reserve
+	return uint64(max)
+}