From 5a08788a05fbcaae305031396d9c512fdaff0bc5 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Thu, 28 Apr 2022 15:31:59 +0200 Subject: [PATCH 1/8] multi: update btcwallet to v0.15.0 Update go.mod to point to latest btcwallet version. --- chainntnfs/test_utils.go | 16 +++++++++------- chainreg/chainregistry.go | 16 +++++++++------- go.mod | 2 +- go.sum | 4 ++-- lnwallet/btcwallet/btcwallet.go | 12 +++++++++++- lnwallet/test/test_interface.go | 16 +++++++++------- routing/chainview/interface_test.go | 16 +++++++++------- 7 files changed, 50 insertions(+), 32 deletions(-) diff --git a/chainntnfs/test_utils.go b/chainntnfs/test_utils.go index 855bfc388..fb5212aa5 100644 --- a/chainntnfs/test_utils.go +++ b/chainntnfs/test_utils.go @@ -235,13 +235,15 @@ func NewBitcoindBackend(t *testing.T, minerAddr string, host := fmt.Sprintf("127.0.0.1:%d", rpcPort) conn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{ - ChainParams: NetParams, - Host: host, - User: "weks", - Pass: "weks", - ZMQBlockHost: zmqBlockHost, - ZMQTxHost: zmqTxHost, - ZMQReadDeadline: 5 * time.Second, + ChainParams: NetParams, + Host: host, + User: "weks", + Pass: "weks", + ZMQConfig: &chain.ZMQConfig{ + ZMQBlockHost: zmqBlockHost, + ZMQTxHost: zmqTxHost, + ZMQReadDeadline: 5 * time.Second, + }, // Fields only required for pruned nodes, not needed for these // tests. Dialer: nil, diff --git a/chainreg/chainregistry.go b/chainreg/chainregistry.go index a19658d24..86b3bd001 100644 --- a/chainreg/chainregistry.go +++ b/chainreg/chainregistry.go @@ -402,13 +402,15 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) { // Establish the connection to bitcoind and create the clients // required for our relevant subsystems. bitcoindConn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{ - ChainParams: cfg.ActiveNetParams.Params, - Host: bitcoindHost, - User: bitcoindMode.RPCUser, - Pass: bitcoindMode.RPCPass, - ZMQBlockHost: bitcoindMode.ZMQPubRawBlock, - ZMQTxHost: bitcoindMode.ZMQPubRawTx, - ZMQReadDeadline: 5 * time.Second, + ChainParams: cfg.ActiveNetParams.Params, + Host: bitcoindHost, + User: bitcoindMode.RPCUser, + Pass: bitcoindMode.RPCPass, + ZMQConfig: &chain.ZMQConfig{ + ZMQBlockHost: bitcoindMode.ZMQPubRawBlock, + ZMQTxHost: bitcoindMode.ZMQPubRawTx, + ZMQReadDeadline: 5 * time.Second, + }, Dialer: cfg.Dialer, PrunedModeMaxPeers: bitcoindMode.PrunedNodeMaxPeers, }) diff --git a/go.mod b/go.mod index 2d06203d8..5d72ce803 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/btcsuite/btcd/btcutil/psbt v1.1.3 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f - github.com/btcsuite/btcwallet v0.14.1-0.20220412233800-3a6d5d0702b7 + github.com/btcsuite/btcwallet v0.15.0 github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3 github.com/btcsuite/btcwallet/wallet/txrules v1.2.0 github.com/btcsuite/btcwallet/walletdb v1.4.0 diff --git a/go.sum b/go.sum index 6e030d2c4..445799fe7 100644 --- a/go.sum +++ b/go.sum @@ -95,8 +95,8 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtyd github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcwallet v0.14.1-0.20220412233800-3a6d5d0702b7 h1:K7omr0FIjwwwQXTMgp2d+DTRH71RlkjRKj/Ir6Zkqb0= -github.com/btcsuite/btcwallet v0.14.1-0.20220412233800-3a6d5d0702b7/go.mod h1:EE9BactCCWhCFoVfxCJrSFINrYqLx/Tq6quxRlPTpzM= +github.com/btcsuite/btcwallet v0.15.0 h1:FdgC7JySVQJIcU+3W+kswDPv8rtzArGNQLOn2g3TiLg= +github.com/btcsuite/btcwallet v0.15.0/go.mod h1:EE9BactCCWhCFoVfxCJrSFINrYqLx/Tq6quxRlPTpzM= github.com/btcsuite/btcwallet/wallet/txauthor v1.2.1/go.mod h1:/74bubxX5Js48d76nf/TsNabpYp/gndUuJw4chzCmhU= github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3 h1:M2yr5UlULvpqtxUqpMxTME/pA92Z9cpqeyvAFk9lAg0= github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3/go.mod h1:T2xSiKGpUkSLCh68aF+FMXmKK9mFqNdHl9VaqOr+JjU= diff --git a/lnwallet/btcwallet/btcwallet.go b/lnwallet/btcwallet/btcwallet.go index c6b8b1a46..f972ab616 100644 --- a/lnwallet/btcwallet/btcwallet.go +++ b/lnwallet/btcwallet/btcwallet.go @@ -932,7 +932,17 @@ func (b *BtcWallet) LeaseOutput(id wtxmgr.LockID, op wire.OutPoint, // ListLeasedOutputs returns a list of all currently locked outputs. func (b *BtcWallet) ListLeasedOutputs() ([]*wtxmgr.LockedOutput, error) { - return b.wallet.ListLeasedOutputs() + leasedOutputs, err := b.wallet.ListLeasedOutputs() + if err != nil { + return nil, err + } + + lockedOutputs := make([]*wtxmgr.LockedOutput, len(leasedOutputs)) + for i, output := range leasedOutputs { + lockedOutputs[i] = output.LockedOutput + } + + return lockedOutputs, nil } // ReleaseOutput unlocks an output, allowing it to be available for coin diff --git a/lnwallet/test/test_interface.go b/lnwallet/test/test_interface.go index 5f0f3fbab..d5e496035 100644 --- a/lnwallet/test/test_interface.go +++ b/lnwallet/test/test_interface.go @@ -3405,13 +3405,15 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver, var chainConn *chain.BitcoindConn err = wait.NoError(func() error { chainConn, err = chain.NewBitcoindConn(&chain.BitcoindConfig{ - ChainParams: netParams, - Host: host, - User: "weks", - Pass: "weks", - ZMQBlockHost: zmqBlockHost, - ZMQTxHost: zmqTxHost, - ZMQReadDeadline: 5 * time.Second, + ChainParams: netParams, + Host: host, + User: "weks", + Pass: "weks", + ZMQConfig: &chain.ZMQConfig{ + ZMQBlockHost: zmqBlockHost, + ZMQTxHost: zmqTxHost, + ZMQReadDeadline: 5 * time.Second, + }, // Fields only required for pruned nodes, not // needed for these tests. Dialer: nil, diff --git a/routing/chainview/interface_test.go b/routing/chainview/interface_test.go index fca3191f4..526967479 100644 --- a/routing/chainview/interface_test.go +++ b/routing/chainview/interface_test.go @@ -828,13 +828,15 @@ var interfaceImpls = []struct { host := fmt.Sprintf("127.0.0.1:%d", rpcPort) chainConn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{ - ChainParams: &chaincfg.RegressionNetParams, - Host: host, - User: "weks", - Pass: "weks", - ZMQBlockHost: zmqBlockHost, - ZMQTxHost: zmqTxHost, - ZMQReadDeadline: 5 * time.Second, + ChainParams: &chaincfg.RegressionNetParams, + Host: host, + User: "weks", + Pass: "weks", + ZMQConfig: &chain.ZMQConfig{ + ZMQBlockHost: zmqBlockHost, + ZMQTxHost: zmqTxHost, + ZMQReadDeadline: 5 * time.Second, + }, // Fields only required for pruned nodes, not // needed for these tests. Dialer: nil, From c76d04ef918777cd4b9b73a03d097ea564ae4f47 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Thu, 28 Apr 2022 15:36:45 +0200 Subject: [PATCH 2/8] multi: make ZmqReadDeadline configurable Make the Bitcoind ZMQReadDeadline option configurable. --- chainreg/chainregistry.go | 2 +- config.go | 5 +++++ lncfg/bitcoind.go | 23 +++++++++++++---------- sample-lnd.conf | 2 ++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/chainreg/chainregistry.go b/chainreg/chainregistry.go index 86b3bd001..030757d45 100644 --- a/chainreg/chainregistry.go +++ b/chainreg/chainregistry.go @@ -409,7 +409,7 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) { ZMQConfig: &chain.ZMQConfig{ ZMQBlockHost: bitcoindMode.ZMQPubRawBlock, ZMQTxHost: bitcoindMode.ZMQPubRawTx, - ZMQReadDeadline: 5 * time.Second, + ZMQReadDeadline: bitcoindMode.ZMQReadDeadline, }, Dialer: cfg.Dialer, PrunedModeMaxPeers: bitcoindMode.PrunedNodeMaxPeers, diff --git a/config.go b/config.go index 42626581f..a18021233 100644 --- a/config.go +++ b/config.go @@ -83,6 +83,10 @@ const ( defaultTorV2PrivateKeyFilename = "v2_onion_private_key" defaultTorV3PrivateKeyFilename = "v3_onion_private_key" + // defaultZMQReadDeadline is the default read deadline to be used for + // both the block and tx ZMQ subscriptions. + defaultZMQReadDeadline = 5 * time.Second + // DefaultAutogenValidity is the default validity of a self-signed // certificate. The value corresponds to 14 months // (14 months * 30 days * 24 hours). @@ -483,6 +487,7 @@ func DefaultConfig() Config { RPCHost: defaultRPCHost, EstimateMode: defaultBitcoindEstimateMode, PrunedNodeMaxPeers: defaultPrunedNodeMaxPeers, + ZMQReadDeadline: defaultZMQReadDeadline, }, Litecoin: &lncfg.Chain{ MinHTLCIn: chainreg.DefaultLitecoinMinHTLCInMSat, diff --git a/lncfg/bitcoind.go b/lncfg/bitcoind.go index 904f9924b..55ec7d44b 100644 --- a/lncfg/bitcoind.go +++ b/lncfg/bitcoind.go @@ -1,16 +1,19 @@ package lncfg +import "time" + // Bitcoind holds the configuration options for the daemon's connection to // bitcoind. type Bitcoind struct { - Dir string `long:"dir" description:"The base directory that contains the node's data, logs, configuration file, etc."` - ConfigPath string `long:"config" description:"Configuration filepath. If not set, will default to the default filename under 'dir'."` - RPCCookie string `long:"rpccookie" description:"Authentication cookie file for RPC connections. If not set, will default to .cookie under 'dir'."` - RPCHost string `long:"rpchost" description:"The daemon's rpc listening address. If a port is omitted, then the default port for the selected chain parameters will be used."` - RPCUser string `long:"rpcuser" description:"Username for RPC connections"` - RPCPass string `long:"rpcpass" default-mask:"-" description:"Password for RPC connections"` - ZMQPubRawBlock string `long:"zmqpubrawblock" description:"The address listening for ZMQ connections to deliver raw block notifications"` - ZMQPubRawTx string `long:"zmqpubrawtx" description:"The address listening for ZMQ connections to deliver raw transaction notifications"` - EstimateMode string `long:"estimatemode" description:"The fee estimate mode. Must be either ECONOMICAL or CONSERVATIVE."` - PrunedNodeMaxPeers int `long:"pruned-node-max-peers" description:"The maximum number of peers lnd will choose from the backend node to retrieve pruned blocks from. This only applies to pruned nodes."` + Dir string `long:"dir" description:"The base directory that contains the node's data, logs, configuration file, etc."` + ConfigPath string `long:"config" description:"Configuration filepath. If not set, will default to the default filename under 'dir'."` + RPCCookie string `long:"rpccookie" description:"Authentication cookie file for RPC connections. If not set, will default to .cookie under 'dir'."` + RPCHost string `long:"rpchost" description:"The daemon's rpc listening address. If a port is omitted, then the default port for the selected chain parameters will be used."` + RPCUser string `long:"rpcuser" description:"Username for RPC connections"` + RPCPass string `long:"rpcpass" default-mask:"-" description:"Password for RPC connections"` + ZMQPubRawBlock string `long:"zmqpubrawblock" description:"The address listening for ZMQ connections to deliver raw block notifications"` + ZMQPubRawTx string `long:"zmqpubrawtx" description:"The address listening for ZMQ connections to deliver raw transaction notifications"` + ZMQReadDeadline time.Duration `long:"zmqreaddeadline" description:"The read deadline for reading ZMQ messages from both the block and tx subscriptions"` + EstimateMode string `long:"estimatemode" description:"The fee estimate mode. Must be either ECONOMICAL or CONSERVATIVE."` + PrunedNodeMaxPeers int `long:"pruned-node-max-peers" description:"The maximum number of peers lnd will choose from the backend node to retrieve pruned blocks from. This only applies to pruned nodes."` } diff --git a/sample-lnd.conf b/sample-lnd.conf index 82ada3de7..3fa45dcb8 100644 --- a/sample-lnd.conf +++ b/sample-lnd.conf @@ -604,6 +604,7 @@ bitcoin.node=btcd ; likely won't need to be set (other than for a remote bitcoind instance). ; bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 ; bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 +; bitcoind.zmqreaddeadline=10s ; Fee estimate mode for bitcoind. It must be either "ECONOMICAL" or "CONSERVATIVE". ; If unset, the default value is "CONSERVATIVE". @@ -811,6 +812,7 @@ litecoin.node=ltcd ; likely won't need to be set (other than for a remote litecoind instance). ; litecoind.zmqpubrawblock=tcp://127.0.0.1:28332 ; litecoind.zmqpubrawtx=tcp://127.0.0.1:28333 +; litecoind.zmqreaddeadline=10s ; Fee estimate mode for litecoind. It must be either "ECONOMICAL" or "CONSERVATIVE". ; If unset, the default value is "CONSERVATIVE". From 33e1f9bfa4c74291b7545d9648f7ac3a877bdcdd Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Wed, 4 May 2022 11:20:22 +0200 Subject: [PATCH 3/8] routing/chainview: add bitcoind rpc polling test Add a new chainview interface test that runds the chainview tests against a bitcoind node that we are getting block and tx notifications from using the rpc interface. --- routing/chainview/interface_test.go | 234 +++++++++++++++++++++++++--- 1 file changed, 214 insertions(+), 20 deletions(-) diff --git a/routing/chainview/interface_test.go b/routing/chainview/interface_test.go index 526967479..bb1700729 100644 --- a/routing/chainview/interface_test.go +++ b/routing/chainview/interface_test.go @@ -4,11 +4,12 @@ import ( "bytes" "fmt" "io/ioutil" - "math/rand" + "net" "os" "os/exec" "path/filepath" "runtime" + "sync/atomic" "testing" "time" @@ -28,6 +29,7 @@ import ( "github.com/lightningnetwork/lnd/blockcache" "github.com/lightningnetwork/lnd/channeldb" "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntest/wait" ) var ( @@ -48,6 +50,35 @@ var ( testScript, _ = txscript.PayToAddrScript(testAddr) ) +var ( + // lastPort is the last port determined to be free for use by a new + // bitcoind server. It should be used atomically. + lastPort uint32 = 1024 +) + +// getFreePort returns the first port that is available for listening by a new +// embedded etcd server. It panics if no port is found and the maximum available +// TCP port is reached. +func getFreePort() int { + port := atomic.AddUint32(&lastPort, 1) + for port < 65535 { + // If there are no errors while attempting to listen on this + // port, close the socket and return it as available. + addr := fmt.Sprintf("127.0.0.1:%d", port) + l, err := net.Listen("tcp4", addr) + if err == nil { + err := l.Close() + if err == nil { + return int(port) + } + } + port = atomic.AddUint32(&lastPort, 1) + } + + // No ports available? Must be a mistake. + panic("no ports available for listening") +} + func waitForMempoolTx(r *rpctest.Harness, txid *chainhash.Hash) error { var found bool var tx *btcutil.Tx @@ -560,9 +591,14 @@ func testFilterBlockDisconnected(node *rpctest.Harness, t.Fatalf("unable to set up mining node: %v", err) } + _, bestHeight, err := reorgNode.Client.GetBestBlock() + if err != nil { + t.Fatalf("error getting best block: %v", err) + } + // Init a chain view that has this node as its block source. cleanUpFunc, reorgView, err := chainViewInit( - reorgNode.RPCConfig(), reorgNode.P2PAddress(), + reorgNode.RPCConfig(), reorgNode.P2PAddress(), bestHeight, ) if err != nil { t.Fatalf("unable to create chain view: %v", err) @@ -751,7 +787,7 @@ func testFilterBlockDisconnected(node *rpctest.Harness, } type chainViewInitFunc func(rpcInfo rpcclient.ConnConfig, - p2pAddr string) (func(), FilteredChainView, error) + p2pAddr string, bestHeight int32) (func(), FilteredChainView, error) type testCase struct { name string @@ -785,7 +821,8 @@ var interfaceImpls = []struct { { name: "bitcoind_zmq", chainViewInit: func(_ rpcclient.ConnConfig, - p2pAddr string) (func(), FilteredChainView, error) { + p2pAddr string, bestHeight int32) (func(), + FilteredChainView, error) { // Start a bitcoind instance. tempBitcoindDir, err := ioutil.TempDir("", "bitcoind") @@ -797,7 +834,7 @@ var interfaceImpls = []struct { cleanUp1 := func() { os.RemoveAll(tempBitcoindDir) } - rpcPort := rand.Int()%(65536-1024) + 1024 + rpcPort := getFreePort() bitcoind := exec.Command( "bitcoind", "-datadir="+tempBitcoindDir, @@ -817,17 +854,23 @@ var interfaceImpls = []struct { cleanUp1() return nil, nil, err } + + // Sanity check to ensure that the process did in fact + // start. + if bitcoind.Process == nil { + cleanUp1() + return nil, nil, fmt.Errorf("bitcoind cmd " + + "Process is not set after Start") + } + cleanUp2 := func() { - bitcoind.Process.Kill() - bitcoind.Wait() + _ = bitcoind.Process.Kill() + _ = bitcoind.Wait() cleanUp1() } - // Wait for the bitcoind instance to start up. - time.Sleep(time.Second) - host := fmt.Sprintf("127.0.0.1:%d", rpcPort) - chainConn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{ + cfg := &chain.BitcoindConfig{ ChainParams: &chaincfg.RegressionNetParams, Host: host, User: "weks", @@ -841,13 +884,140 @@ var interfaceImpls = []struct { // needed for these tests. Dialer: nil, PrunedModeMaxPeers: 0, - }) + } + + var chainConn *chain.BitcoindConn + err = wait.NoError(func() error { + chainConn, err = chain.NewBitcoindConn(cfg) + if err != nil { + return err + } + + err = chainConn.Start() + if err != nil { + return err + } + + client := chainConn.NewBitcoindClient() + _, currentHeight, err := client.GetBestBlock() + if err != nil { + return err + } + + if currentHeight < bestHeight { + return fmt.Errorf("not synced yet") + } + + return nil + }, 10*time.Second) if err != nil { return cleanUp2, nil, fmt.Errorf("unable to "+ "establish connection to bitcoind: %v", err) } - if err := chainConn.Start(); err != nil { + cleanUp3 := func() { + chainConn.Stop() + cleanUp2() + } + + blockCache := blockcache.NewBlockCache(10000) + + chainView := NewBitcoindFilteredChainView( + chainConn, blockCache, + ) + + return cleanUp3, chainView, nil + }, + }, + { + name: "bitcoind_polling", + chainViewInit: func(_ rpcclient.ConnConfig, + p2pAddr string, bestHeight int32) (func(), + FilteredChainView, error) { + + // Start a bitcoind instance. + tempBitcoindDir, err := ioutil.TempDir("", "bitcoind") + if err != nil { + return nil, nil, err + } + cleanUp1 := func() { + os.RemoveAll(tempBitcoindDir) + } + rpcPort := getFreePort() + bitcoind := exec.Command( + "bitcoind", + "-datadir="+tempBitcoindDir, + "-regtest", + "-connect="+p2pAddr, + "-txindex", + "-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6f"+ + "d$507c670e800a95284294edb5773b05544b"+ + "220110063096c221be9933c82d38e1", + fmt.Sprintf("-rpcport=%d", rpcPort), + "-disablewallet", + ) + err = bitcoind.Start() + if err != nil { + cleanUp1() + return nil, nil, err + } + + // Sanity check to ensure that the process did in fact + // start. + if bitcoind.Process == nil { + cleanUp1() + return nil, nil, fmt.Errorf("bitcoind cmd " + + "Process is not set after Start") + } + + cleanUp2 := func() { + _ = bitcoind.Process.Kill() + _ = bitcoind.Wait() + cleanUp1() + } + + host := fmt.Sprintf("127.0.0.1:%d", rpcPort) + cfg := &chain.BitcoindConfig{ + ChainParams: &chaincfg.RegressionNetParams, + Host: host, + User: "weks", + Pass: "weks", + PollingConfig: &chain.PollingConfig{ + BlockPollingInterval: time.Millisecond * 100, + TxPollingInterval: time.Millisecond * 100, + }, + // Fields only required for pruned nodes, not + // needed for these tests. + Dialer: nil, + PrunedModeMaxPeers: 0, + } + + // Wait for the bitcoind instance to start up. + var chainConn *chain.BitcoindConn + err = wait.NoError(func() error { + chainConn, err = chain.NewBitcoindConn(cfg) + if err != nil { + return err + } + + err = chainConn.Start() + if err != nil { + return err + } + + client := chainConn.NewBitcoindClient() + _, currentHeight, err := client.GetBestBlock() + if err != nil { + return err + } + + if currentHeight < bestHeight { + return fmt.Errorf("not synced yet") + } + + return nil + }, 10*time.Second) + if err != nil { return cleanUp2, nil, fmt.Errorf("unable to "+ "establish connection to bitcoind: %v", err) @@ -869,7 +1039,8 @@ var interfaceImpls = []struct { { name: "p2p_neutrino", chainViewInit: func(_ rpcclient.ConnConfig, - p2pAddr string) (func(), FilteredChainView, error) { + p2pAddr string, bestHeight int32) (func(), + FilteredChainView, error) { spvDir, err := ioutil.TempDir("", "neutrino") if err != nil { @@ -895,12 +1066,30 @@ var interfaceImpls = []struct { if err != nil { return nil, nil, err } - spvNode.Start() // Wait until the node has fully synced up to the local // btcd node. - for !spvNode.IsCurrent() { - time.Sleep(time.Millisecond * 100) + err = wait.NoError(func() error { + err := spvNode.Start() + if err != nil { + return err + } + + bestBlock, err := spvNode.BestBlock() + if err != nil { + return err + } + + if bestBlock.Height < bestHeight { + return fmt.Errorf("not synced yet") + } + + return nil + }, 10*time.Second) + if err != nil { + return nil, nil, fmt.Errorf("unable to "+ + "establish connection to bitcoind: %v", + err) } cleanUp := func() { @@ -924,7 +1113,8 @@ var interfaceImpls = []struct { { name: "btcd_websockets", chainViewInit: func(config rpcclient.ConnConfig, - _ string) (func(), FilteredChainView, error) { + p2pAddr string, bestHeight int32) (func(), + FilteredChainView, error) { blockCache := blockcache.NewBlockCache(10000) chainView, err := NewBtcdFilteredChainView( @@ -960,13 +1150,17 @@ func TestFilteredChainView(t *testing.T) { t.Logf("Testing '%v' implementation of FilteredChainView", chainViewImpl.name) + _, bestHeight, err := miner.Client.GetBestBlock() + if err != nil { + t.Fatalf("error getting best block: %v", err) + } + cleanUpFunc, chainView, err := chainViewImpl.chainViewInit( - rpcConfig, p2pAddr, + rpcConfig, p2pAddr, bestHeight, ) if err != nil { t.Fatalf("unable to make chain view: %v", err) } - if err := chainView.Start(); err != nil { t.Fatalf("unable to start chain view: %v", err) } From f766268e8c7cf9dd332aaac55c45e9be999cc39d Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Wed, 4 May 2022 11:23:58 +0200 Subject: [PATCH 4/8] lnwallet/test: test against bitcoind-rpc-polling backend --- lnwallet/test/bitcoind/bitcoind_test.go | 12 ++- lnwallet/test/test_interface.go | 119 +++++++++++++++++++++++- 2 files changed, 124 insertions(+), 7 deletions(-) diff --git a/lnwallet/test/bitcoind/bitcoind_test.go b/lnwallet/test/bitcoind/bitcoind_test.go index a5df25281..b63ff9258 100644 --- a/lnwallet/test/bitcoind/bitcoind_test.go +++ b/lnwallet/test/bitcoind/bitcoind_test.go @@ -6,8 +6,14 @@ import ( lnwallettest "github.com/lightningnetwork/lnd/lnwallet/test" ) -// TestLightningWallet tests LightningWallet powered by bitcoind against our -// suite of interface tests. -func TestLightningWallet(t *testing.T) { +// TestLightningWalletBitcoindZMQ tests LightningWallet powered by bitcoind, +// using its ZMQ interface, against our suite of interface tests. +func TestLightningWalletBitcoindZMQ(t *testing.T) { lnwallettest.TestLightningWallet(t, "bitcoind") } + +// TestLightningWalletBitcoindRPCPolling tests LightningWallet powered by +// bitcoind, using its RPC interface, against our suite of interface tests. +func TestLightningWalletBitcoindRPCPolling(t *testing.T) { + lnwallettest.TestLightningWallet(t, "bitcoind-rpc-polling") +} diff --git a/lnwallet/test/test_interface.go b/lnwallet/test/test_interface.go index d5e496035..2a99c7287 100644 --- a/lnwallet/test/test_interface.go +++ b/lnwallet/test/test_interface.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "fmt" "io/ioutil" - "math/rand" "net" "os" "os/exec" @@ -14,6 +13,7 @@ import ( "reflect" "runtime" "strings" + "sync/atomic" "testing" "time" @@ -96,6 +96,35 @@ var ( defaultMaxLocalCsvDelay uint16 = 10000 ) +var ( + // lastPort is the last port determined to be free for use by a new + // bitcoind server. It should be used atomically. + lastPort uint32 = 1024 +) + +// getFreePort returns the first port that is available for listening by a new +// embedded etcd server. It panics if no port is found and the maximum available +// TCP port is reached. +func getFreePort() int { + port := atomic.AddUint32(&lastPort, 1) + for port < 65535 { + // If there are no errors while attempting to listen on this + // port, close the socket and return it as available. + addr := fmt.Sprintf("127.0.0.1:%d", port) + l, err := net.Listen("tcp4", addr) + if err == nil { + err := l.Close() + if err == nil { + return int(port) + } + } + port = atomic.AddUint32(&lastPort, 1) + } + + // No ports available? Must be a mistake. + panic("no ports available for listening") +} + // assertProperBalance asserts than the total value of the unspent outputs // within the wallet are *exactly* amount. If unable to retrieve the current // balance, or the assertion fails, the test will halt with a fatal error. @@ -3377,7 +3406,7 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver, zmqBlockHost := "ipc:///" + tempBitcoindDir + "/blocks.socket" zmqTxHost := "ipc:///" + tempBitcoindDir + "/tx.socket" defer os.RemoveAll(tempBitcoindDir) - rpcPort := rand.Int()%(65536-1024) + 1024 + rpcPort := getFreePort() bitcoind := exec.Command( "bitcoind", "-datadir="+tempBitcoindDir, @@ -3396,8 +3425,18 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver, if err != nil { t.Fatalf("couldn't start bitcoind: %v", err) } - defer bitcoind.Wait() - defer bitcoind.Process.Kill() + + // Sanity check to ensure that the process did in fact + // start. + if bitcoind.Process == nil { + t.Fatalf("bitcoind cmd Process is not set " + + "after Start") + } + + defer func() { + _ = bitcoind.Process.Kill() + _ = bitcoind.Wait() + }() // Wait for the bitcoind instance to start up. @@ -3435,6 +3474,78 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver, // Bob. aliceClient = chainConn.NewBitcoindClient() bobClient = chainConn.NewBitcoindClient() + + case "bitcoind-rpc-polling": + // Start a bitcoind instance. + tempBitcoindDir, err := ioutil.TempDir("", "bitcoind") + if err != nil { + t.Fatalf("unable to create temp directory: %v", err) + } + defer os.RemoveAll(tempBitcoindDir) + rpcPort := getFreePort() + bitcoind := exec.Command( + "bitcoind", + "-datadir="+tempBitcoindDir, + "-regtest", + "-connect="+miningNode.P2PAddress(), + "-txindex", + "-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6f"+ + "d$507c670e800a95284294edb5773b05544b"+ + "220110063096c221be9933c82d38e1", + fmt.Sprintf("-rpcport=%d", rpcPort), + "-disablewallet", + ) + err = bitcoind.Start() + if err != nil { + t.Fatalf("couldn't start bitcoind: %v", err) + } + defer func() { + _ = bitcoind.Process.Kill() + _ = bitcoind.Wait() + }() + + // Sanity check to ensure that the process did in fact + // start. + if bitcoind.Process == nil { + t.Fatalf("bitcoind cmd Process is not set " + + "after Start") + } + + // Wait for the bitcoind instance to start up. + host := fmt.Sprintf("127.0.0.1:%d", rpcPort) + var chainConn *chain.BitcoindConn + err = wait.NoError(func() error { + chainConn, err = chain.NewBitcoindConn(&chain.BitcoindConfig{ + ChainParams: netParams, + Host: host, + User: "weks", + Pass: "weks", + PollingConfig: &chain.PollingConfig{ + BlockPollingInterval: time.Millisecond * 20, + TxPollingInterval: time.Millisecond * 20, + }, + // Fields only required for pruned nodes, not + // needed for these tests. + Dialer: nil, + PrunedModeMaxPeers: 0, + }) + if err != nil { + return err + } + + return chainConn.Start() + }, 10*time.Second) + if err != nil { + t.Fatalf("unable to establish connection to "+ + "bitcoind: %v", err) + } + defer chainConn.Stop() + + // Create a btcwallet bitcoind client for both Alice and + // Bob. + aliceClient = chainConn.NewBitcoindClient() + bobClient = chainConn.NewBitcoindClient() + default: t.Fatalf("unknown chain driver: %v", backEnd) } From 0459ccf5422b6f910777cc573e4b44839a293704 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Wed, 4 May 2022 11:29:11 +0200 Subject: [PATCH 5/8] chainntnfs: test both zmq and rpc bitcoind modes --- chainntnfs/bitcoindnotify/bitcoind.go | 11 +++-- chainntnfs/bitcoindnotify/bitcoind_test.go | 14 +++++- chainntnfs/bitcoindnotify/driver.go | 18 +++++--- chainntnfs/test/bitcoind/bitcoind_test.go | 1 + chainntnfs/test/test_interface.go | 14 +++++- chainntnfs/test_utils.go | 51 ++++++++++++++-------- 6 files changed, 79 insertions(+), 30 deletions(-) diff --git a/chainntnfs/bitcoindnotify/bitcoind.go b/chainntnfs/bitcoindnotify/bitcoind.go index 016d46dc4..a83b26e06 100644 --- a/chainntnfs/bitcoindnotify/bitcoind.go +++ b/chainntnfs/bitcoindnotify/bitcoind.go @@ -19,9 +19,14 @@ import ( ) const ( - // notifierType uniquely identifies this concrete implementation of the - // ChainNotifier interface. - notifierType = "bitcoind" + // notifierType uniquely identifies a concrete implementation of the + // ChainNotifier interface that makes use of the bitcoind ZMQ interface. + notifierTypeZMQ = "bitcoind" + + // notifierTypeRPCPolling uniquely identifies a concrete implementation + // of the ChainNotifier interface that makes use of the bitcoind RPC + // interface. + notifierTypeRPCPolling = "bitcoind-rpc-polling" ) // TODO(roasbeef): generalize struct below: diff --git a/chainntnfs/bitcoindnotify/bitcoind_test.go b/chainntnfs/bitcoindnotify/bitcoind_test.go index 796735668..f9511ffe0 100644 --- a/chainntnfs/bitcoindnotify/bitcoind_test.go +++ b/chainntnfs/bitcoindnotify/bitcoind_test.go @@ -108,13 +108,18 @@ func syncNotifierWithMiner(t *testing.T, notifier *BitcoindNotifier, // TestHistoricalConfDetailsTxIndex ensures that we correctly retrieve // historical confirmation details using the backend node's txindex. func TestHistoricalConfDetailsTxIndex(t *testing.T) { + testHistoricalConfDetailsTxIndex(t, true) + testHistoricalConfDetailsTxIndex(t, false) +} + +func testHistoricalConfDetailsTxIndex(t *testing.T, rpcPolling bool) { miner, tearDown := chainntnfs.NewMiner( t, []string{"--txindex"}, true, 25, ) defer tearDown() bitcoindConn, cleanUp := chainntnfs.NewBitcoindBackend( - t, miner.P2PAddress(), true, + t, miner.P2PAddress(), true, rpcPolling, ) defer cleanUp() @@ -206,11 +211,16 @@ func TestHistoricalConfDetailsTxIndex(t *testing.T) { // historical confirmation details using the set of fallback methods when the // backend node's txindex is disabled. func TestHistoricalConfDetailsNoTxIndex(t *testing.T) { + testHistoricalConfDetailsNoTxIndex(t, true) + testHistoricalConfDetailsNoTxIndex(t, false) +} + +func testHistoricalConfDetailsNoTxIndex(t *testing.T, rpcpolling bool) { miner, tearDown := chainntnfs.NewMiner(t, nil, true, 25) defer tearDown() bitcoindConn, cleanUp := chainntnfs.NewBitcoindBackend( - t, miner.P2PAddress(), false, + t, miner.P2PAddress(), false, rpcpolling, ) defer cleanUp() diff --git a/chainntnfs/bitcoindnotify/driver.go b/chainntnfs/bitcoindnotify/driver.go index 634aa3545..1968f74c6 100644 --- a/chainntnfs/bitcoindnotify/driver.go +++ b/chainntnfs/bitcoindnotify/driver.go @@ -56,13 +56,21 @@ func createNewNotifier(args ...interface{}) (chainntnfs.ChainNotifier, error) { // chainntnfs.ChainNotifier interface. func init() { // Register the driver. - notifier := &chainntnfs.NotifierDriver{ - NotifierType: notifierType, + notifierZMQ := &chainntnfs.NotifierDriver{ + NotifierType: notifierTypeZMQ, New: createNewNotifier, } - - if err := chainntnfs.RegisterNotifier(notifier); err != nil { + if err := chainntnfs.RegisterNotifier(notifierZMQ); err != nil { panic(fmt.Sprintf("failed to register notifier driver '%s': %v", - notifierType, err)) + notifierTypeZMQ, err)) + } + + notifierRPC := &chainntnfs.NotifierDriver{ + NotifierType: notifierTypeRPCPolling, + New: createNewNotifier, + } + if err := chainntnfs.RegisterNotifier(notifierRPC); err != nil { + panic(fmt.Sprintf("failed to register notifier driver '%s': %v", + notifierTypeRPCPolling, err)) } } diff --git a/chainntnfs/test/bitcoind/bitcoind_test.go b/chainntnfs/test/bitcoind/bitcoind_test.go index 32335e7bb..38779f8c7 100644 --- a/chainntnfs/test/bitcoind/bitcoind_test.go +++ b/chainntnfs/test/bitcoind/bitcoind_test.go @@ -13,4 +13,5 @@ import ( // powered chain notifier. func TestInterfaces(t *testing.T) { chainntnfstest.TestInterfaces(t, "bitcoind") + chainntnfstest.TestInterfaces(t, "bitcoind-rpc-polling") } diff --git a/chainntnfs/test/test_interface.go b/chainntnfs/test/test_interface.go index 70f79b4bc..291d72a56 100644 --- a/chainntnfs/test/test_interface.go +++ b/chainntnfs/test/test_interface.go @@ -1966,7 +1966,19 @@ func TestInterfaces(t *testing.T, targetBackEnd string) { case "bitcoind": var bitcoindConn *chain.BitcoindConn bitcoindConn, cleanUp = chainntnfs.NewBitcoindBackend( - t, p2pAddr, true, + t, p2pAddr, true, false, + ) + newNotifier = func() (chainntnfs.TestChainNotifier, error) { + return bitcoindnotify.New( + bitcoindConn, chainntnfs.NetParams, + hintCache, hintCache, blockCache, + ), nil + } + + case "bitcoind-rpc-polling": + var bitcoindConn *chain.BitcoindConn + bitcoindConn, cleanUp = chainntnfs.NewBitcoindBackend( + t, p2pAddr, true, true, ) newNotifier = func() (chainntnfs.TestChainNotifier, error) { return bitcoindnotify.New( diff --git a/chainntnfs/test_utils.go b/chainntnfs/test_utils.go index fb5212aa5..65eaeb072 100644 --- a/chainntnfs/test_utils.go +++ b/chainntnfs/test_utils.go @@ -26,6 +26,7 @@ import ( "github.com/btcsuite/btcwallet/walletdb" "github.com/lightninglabs/neutrino" "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntest/wait" ) var ( @@ -193,10 +194,12 @@ func NewMiner(t *testing.T, extraArgs []string, createChain bool, // NewBitcoindBackend spawns a new bitcoind node that connects to a miner at the // specified address. The txindex boolean can be set to determine whether the -// backend node should maintain a transaction index. A connection to the newly -// spawned bitcoind node is returned. -func NewBitcoindBackend(t *testing.T, minerAddr string, - txindex bool) (*chain.BitcoindConn, func()) { +// backend node should maintain a transaction index. The rpcpolling boolean +// can be set to determine whether bitcoind's RPC polling interface should be +// used for block and tx notifications or if its ZMQ interface should be used. +// A connection to the newly spawned bitcoind node is returned. +func NewBitcoindBackend(t *testing.T, minerAddr string, txindex, + rpcpolling bool) (*chain.BitcoindConn, func()) { t.Helper() @@ -231,31 +234,41 @@ func NewBitcoindBackend(t *testing.T, minerAddr string, } // Wait for the bitcoind instance to start up. - time.Sleep(time.Second) - host := fmt.Sprintf("127.0.0.1:%d", rpcPort) - conn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{ + cfg := &chain.BitcoindConfig{ ChainParams: NetParams, Host: host, User: "weks", Pass: "weks", - ZMQConfig: &chain.ZMQConfig{ - ZMQBlockHost: zmqBlockHost, - ZMQTxHost: zmqTxHost, - ZMQReadDeadline: 5 * time.Second, - }, // Fields only required for pruned nodes, not needed for these // tests. Dialer: nil, PrunedModeMaxPeers: 0, - }) - if err != nil { - bitcoind.Process.Kill() - bitcoind.Wait() - os.RemoveAll(tempBitcoindDir) - t.Fatalf("unable to establish connection to bitcoind: %v", err) } - if err := conn.Start(); err != nil { + + if rpcpolling { + cfg.ZMQConfig = &chain.ZMQConfig{ + ZMQBlockHost: zmqBlockHost, + ZMQTxHost: zmqTxHost, + ZMQReadDeadline: 5 * time.Second, + } + } else { + cfg.PollingConfig = &chain.PollingConfig{ + BlockPollingInterval: time.Millisecond * 20, + TxPollingInterval: time.Millisecond * 20, + } + } + + var conn *chain.BitcoindConn + err = wait.NoError(func() error { + conn, err = chain.NewBitcoindConn(cfg) + if err != nil { + return err + } + + return conn.Start() + }, 10*time.Second) + if err != nil { bitcoind.Process.Kill() bitcoind.Wait() os.RemoveAll(tempBitcoindDir) From 7509af1a705f40b4999c9efe00de0f3c72a8d251 Mon Sep 17 00:00:00 2001 From: Orbital Date: Thu, 10 Mar 2022 10:01:44 +0000 Subject: [PATCH 6/8] multi: add rpcpolling config options --- chainreg/chainregistry.go | 34 ++++++++++++++++++++++------------ lncfg/bitcoind.go | 25 ++++++++++++++----------- sample-lnd.conf | 16 ++++++++++++++++ 3 files changed, 52 insertions(+), 23 deletions(-) diff --git a/chainreg/chainregistry.go b/chainreg/chainregistry.go index 030757d45..c2d44d6e8 100644 --- a/chainreg/chainregistry.go +++ b/chainreg/chainregistry.go @@ -399,21 +399,31 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) { } } - // Establish the connection to bitcoind and create the clients - // required for our relevant subsystems. - bitcoindConn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{ - ChainParams: cfg.ActiveNetParams.Params, - Host: bitcoindHost, - User: bitcoindMode.RPCUser, - Pass: bitcoindMode.RPCPass, - ZMQConfig: &chain.ZMQConfig{ + bitcoindCfg := &chain.BitcoindConfig{ + ChainParams: cfg.ActiveNetParams.Params, + Host: bitcoindHost, + User: bitcoindMode.RPCUser, + Pass: bitcoindMode.RPCPass, + Dialer: cfg.Dialer, + PrunedModeMaxPeers: bitcoindMode.PrunedNodeMaxPeers, + } + + if bitcoindMode.RPCPolling { + bitcoindCfg.PollingConfig = &chain.PollingConfig{ + BlockPollingInterval: bitcoindMode.BlockPollingInterval, + TxPollingInterval: bitcoindMode.TxPollingInterval, + } + } else { + bitcoindCfg.ZMQConfig = &chain.ZMQConfig{ ZMQBlockHost: bitcoindMode.ZMQPubRawBlock, ZMQTxHost: bitcoindMode.ZMQPubRawTx, ZMQReadDeadline: bitcoindMode.ZMQReadDeadline, - }, - Dialer: cfg.Dialer, - PrunedModeMaxPeers: bitcoindMode.PrunedNodeMaxPeers, - }) + } + } + + // Establish the connection to bitcoind and create the clients + // required for our relevant subsystems. + bitcoindConn, err := chain.NewBitcoindConn(bitcoindCfg) if err != nil { return nil, nil, err } diff --git a/lncfg/bitcoind.go b/lncfg/bitcoind.go index 55ec7d44b..cf0f00232 100644 --- a/lncfg/bitcoind.go +++ b/lncfg/bitcoind.go @@ -5,15 +5,18 @@ import "time" // Bitcoind holds the configuration options for the daemon's connection to // bitcoind. type Bitcoind struct { - Dir string `long:"dir" description:"The base directory that contains the node's data, logs, configuration file, etc."` - ConfigPath string `long:"config" description:"Configuration filepath. If not set, will default to the default filename under 'dir'."` - RPCCookie string `long:"rpccookie" description:"Authentication cookie file for RPC connections. If not set, will default to .cookie under 'dir'."` - RPCHost string `long:"rpchost" description:"The daemon's rpc listening address. If a port is omitted, then the default port for the selected chain parameters will be used."` - RPCUser string `long:"rpcuser" description:"Username for RPC connections"` - RPCPass string `long:"rpcpass" default-mask:"-" description:"Password for RPC connections"` - ZMQPubRawBlock string `long:"zmqpubrawblock" description:"The address listening for ZMQ connections to deliver raw block notifications"` - ZMQPubRawTx string `long:"zmqpubrawtx" description:"The address listening for ZMQ connections to deliver raw transaction notifications"` - ZMQReadDeadline time.Duration `long:"zmqreaddeadline" description:"The read deadline for reading ZMQ messages from both the block and tx subscriptions"` - EstimateMode string `long:"estimatemode" description:"The fee estimate mode. Must be either ECONOMICAL or CONSERVATIVE."` - PrunedNodeMaxPeers int `long:"pruned-node-max-peers" description:"The maximum number of peers lnd will choose from the backend node to retrieve pruned blocks from. This only applies to pruned nodes."` + Dir string `long:"dir" description:"The base directory that contains the node's data, logs, configuration file, etc."` + ConfigPath string `long:"config" description:"Configuration filepath. If not set, will default to the default filename under 'dir'."` + RPCCookie string `long:"rpccookie" description:"Authentication cookie file for RPC connections. If not set, will default to .cookie under 'dir'."` + RPCHost string `long:"rpchost" description:"The daemon's rpc listening address. If a port is omitted, then the default port for the selected chain parameters will be used."` + RPCUser string `long:"rpcuser" description:"Username for RPC connections"` + RPCPass string `long:"rpcpass" default-mask:"-" description:"Password for RPC connections"` + ZMQPubRawBlock string `long:"zmqpubrawblock" description:"The address listening for ZMQ connections to deliver raw block notifications"` + ZMQPubRawTx string `long:"zmqpubrawtx" description:"The address listening for ZMQ connections to deliver raw transaction notifications"` + ZMQReadDeadline time.Duration `long:"zmqreaddeadline" description:"The read deadline for reading ZMQ messages from both the block and tx subscriptions"` + EstimateMode string `long:"estimatemode" description:"The fee estimate mode. Must be either ECONOMICAL or CONSERVATIVE."` + PrunedNodeMaxPeers int `long:"pruned-node-max-peers" description:"The maximum number of peers lnd will choose from the backend node to retrieve pruned blocks from. This only applies to pruned nodes."` + RPCPolling bool `long:"rpcpolling" description:"Poll the bitcoind RPC interface for block and transaction notifications instead of using the ZMQ interface"` + BlockPollingInterval time.Duration `long:"blockpollinginterval" description:"The interval that will be used to poll bitcoind for new blocks. Only used if rpcpolling is true."` + TxPollingInterval time.Duration `long:"txpollinginterval" description:"The interval that will be used to poll bitcoind for new tx. Only used if rpcpolling is true."` } diff --git a/sample-lnd.conf b/sample-lnd.conf index 3fa45dcb8..114ad5c19 100644 --- a/sample-lnd.conf +++ b/sample-lnd.conf @@ -606,6 +606,14 @@ bitcoin.node=btcd ; bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 ; bitcoind.zmqreaddeadline=10s +; Use bitcoind's rpc interface to get block and transaction notifications +; instead of using the zmq interface. Only the rpcpolling option needs to +; be set in order to enable this, the rest of the options can be used to +; change the default values used for this configuration. +; bitcoind.rpcpolling +; bitcoind.blockpollinginterval=1m +; bitcoind.txpollinginterval=30s + ; Fee estimate mode for bitcoind. It must be either "ECONOMICAL" or "CONSERVATIVE". ; If unset, the default value is "CONSERVATIVE". ; bitcoind.estimatemode=CONSERVATIVE @@ -814,6 +822,14 @@ litecoin.node=ltcd ; litecoind.zmqpubrawtx=tcp://127.0.0.1:28333 ; litecoind.zmqreaddeadline=10s +; Use litecoind's rpc interface to get block and transaction notifications +; instead of using the zmq interface. Only the rpcpolling option needs to +; be set in order to enable this, the rest of the options can be used to +; change the default values used for this configuration. +; litecoind.rpcpolling +; litecoind.blockpollinginterval=1m +; litecoind.txpollinginterval=30s + ; Fee estimate mode for litecoind. It must be either "ECONOMICAL" or "CONSERVATIVE". ; If unset, the default value is "CONSERVATIVE". ; litecoind.estimatemode=CONSERVATIVE From e789107e1f78f0376cecdc6ee78b69e29cddcb15 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Thu, 10 Mar 2022 10:32:32 +0000 Subject: [PATCH 7/8] multi: new bitcoind rpcpolling backend for itests --- .github/workflows/main.yml | 2 ++ chainreg/chainregistry.go | 2 +- config.go | 18 +++++++++++++----- lntest/bitcoind.go | 6 +++--- lntest/bitcoind_common.go | 23 +++++++++++++++++------ lntest/bitcoind_notxindex.go | 6 +++--- lntest/bitcoind_rpcpolling.go | 23 +++++++++++++++++++++++ 7 files changed, 62 insertions(+), 18 deletions(-) create mode 100644 lntest/bitcoind_rpcpolling.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b0217e611..5e568885a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -259,6 +259,8 @@ jobs: args: backend=bitcoind - name: bitcoind-notxindex args: backend="bitcoind notxindex" + - name: bitcoind-rpcpolling + args: backend="bitcoind rpcpolling" - name: bitcoind-etcd args: backend=bitcoind dbbackend=etcd - name: bitcoind-postgres diff --git a/chainreg/chainregistry.go b/chainreg/chainregistry.go index c2d44d6e8..bd6f34b0b 100644 --- a/chainreg/chainregistry.go +++ b/chainreg/chainregistry.go @@ -507,7 +507,7 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) { // version 0.17.0) we make sure lnd subscribes to the correct // zmq events. We do this to avoid a situation in which we are // not notified of new transactions or blocks. - if ver >= 170000 { + if ver >= 170000 && !bitcoindMode.RPCPolling { zmqPubRawBlockURL, err := url.Parse(bitcoindMode.ZMQPubRawBlock) if err != nil { return nil, nil, err diff --git a/config.go b/config.go index a18021233..48cc3f16b 100644 --- a/config.go +++ b/config.go @@ -1777,11 +1777,19 @@ func parseRPCParams(cConfig *lncfg.Chain, nodeConfig interface{}, } } - // If all of RPCUser, RPCPass, ZMQBlockHost, and ZMQTxHost are - // set, we assume those parameters are good to use. - if conf.RPCUser != "" && conf.RPCPass != "" && - conf.ZMQPubRawBlock != "" && conf.ZMQPubRawTx != "" { - return nil + if conf.RPCUser != "" && conf.RPCPass != "" { + // If all of RPCUser, RPCPass, ZMQBlockHost, and + // ZMQTxHost are set, we assume those parameters are + // good to use. + if conf.ZMQPubRawBlock != "" && conf.ZMQPubRawTx != "" { + return nil + } + + // If RPCUser and RPCPass are set and RPCPolling is + // enabled, we assume the parameters are good to use. + if conf.RPCPolling { + return nil + } } // Get the daemon name for displaying proper errors. diff --git a/lntest/bitcoind.go b/lntest/bitcoind.go index 37ff5fb5d..37800169b 100644 --- a/lntest/bitcoind.go +++ b/lntest/bitcoind.go @@ -1,5 +1,5 @@ -//go:build bitcoind && !notxindex -// +build bitcoind,!notxindex +//go:build bitcoind && !notxindex && !rpcpolling +// +build bitcoind,!notxindex,!rpcpolling package lntest @@ -19,5 +19,5 @@ func NewBackend(miner string, netParams *chaincfg.Params) ( "-disablewallet", } - return newBackend(miner, netParams, extraArgs) + return newBackend(miner, netParams, extraArgs, false) } diff --git a/lntest/bitcoind_common.go b/lntest/bitcoind_common.go index 937aca540..d9bf102f6 100644 --- a/lntest/bitcoind_common.go +++ b/lntest/bitcoind_common.go @@ -29,6 +29,7 @@ type BitcoindBackendConfig struct { zmqTxPath string p2pPort int rpcClient *rpcclient.Client + rpcPolling bool // minerAddr is the p2p address of the miner to connect to. minerAddr string @@ -46,10 +47,19 @@ func (b BitcoindBackendConfig) GenArgs() []string { args = append(args, fmt.Sprintf("--bitcoind.rpchost=%v", b.rpcHost)) args = append(args, fmt.Sprintf("--bitcoind.rpcuser=%v", b.rpcUser)) args = append(args, fmt.Sprintf("--bitcoind.rpcpass=%v", b.rpcPass)) - args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawblock=%v", - b.zmqBlockPath)) - args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawtx=%v", - b.zmqTxPath)) + + if b.rpcPolling { + args = append(args, fmt.Sprintf("--bitcoind.rpcpolling")) + args = append(args, + fmt.Sprintf("--bitcoind.blockpollinginterval=10ms")) + args = append(args, + fmt.Sprintf("--bitcoind.txpollinginterval=10ms")) + } else { + args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawblock=%v", + b.zmqBlockPath)) + args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawtx=%v", + b.zmqTxPath)) + } return args } @@ -76,8 +86,8 @@ func (b BitcoindBackendConfig) Name() string { // newBackend starts a bitcoind node with the given extra parameters and returns // a BitcoindBackendConfig for that node. -func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) ( - *BitcoindBackendConfig, func() error, error) { +func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string, + rpcPolling bool) (*BitcoindBackendConfig, func() error, error) { baseLogDir := fmt.Sprintf(logDirPattern, GetLogDir()) if netParams != &chaincfg.RegressionNetParams { @@ -192,6 +202,7 @@ func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) ( p2pPort: p2pPort, rpcClient: client, minerAddr: miner, + rpcPolling: rpcPolling, } return &bd, cleanUp, nil diff --git a/lntest/bitcoind_notxindex.go b/lntest/bitcoind_notxindex.go index 2aa7e4a63..de7959eba 100644 --- a/lntest/bitcoind_notxindex.go +++ b/lntest/bitcoind_notxindex.go @@ -1,5 +1,5 @@ -//go:build bitcoind && notxindex -// +build bitcoind,notxindex +//go:build bitcoind && notxindex && !rpcpolling +// +build bitcoind,notxindex,!rpcpolling package lntest @@ -18,5 +18,5 @@ func NewBackend(miner string, netParams *chaincfg.Params) ( "-disablewallet", } - return newBackend(miner, netParams, extraArgs) + return newBackend(miner, netParams, extraArgs, false) } diff --git a/lntest/bitcoind_rpcpolling.go b/lntest/bitcoind_rpcpolling.go new file mode 100644 index 000000000..ca203ad2c --- /dev/null +++ b/lntest/bitcoind_rpcpolling.go @@ -0,0 +1,23 @@ +//go:build bitcoind && rpcpolling +// +build bitcoind,rpcpolling + +package lntest + +import ( + "github.com/btcsuite/btcd/chaincfg" +) + +// NewBackend starts a bitcoind node without the txindex enabled and returns a +// BitoindBackendConfig for that node. +func NewBackend(miner string, netParams *chaincfg.Params) ( + *BitcoindBackendConfig, func() error, error) { + + extraArgs := []string{ + "-debug", + "-regtest", + "-txindex", + "-disablewallet", + } + + return newBackend(miner, netParams, extraArgs, true) +} From a4bd3892d2df85ad8b34fa1a27ef0094dd2787f8 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Fri, 18 Mar 2022 14:37:28 +0200 Subject: [PATCH 8/8] docs: update release notes --- docs/release-notes/release-notes-0.15.0.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/release-notes/release-notes-0.15.0.md b/docs/release-notes/release-notes-0.15.0.md index 7c26e4199..5dbd76c90 100644 --- a/docs/release-notes/release-notes-0.15.0.md +++ b/docs/release-notes/release-notes-0.15.0.md @@ -64,6 +64,12 @@ to Bitcoin nodes that advertise a Tor v3 onion service address. capable of status checks, adding, disconnecting and listing peers, fetching compact filters and block/block headers. +## Btcwallet + +* [Add option to configure the block and transaction subscription + notifications from bitcoind to be obtained through polling of the RPC + interface instead of using ZMQ](https://github.com/lightningnetwork/lnd/pull/6345) + ## Bug Fixes * [Pipelining an UpdateFulfillHTLC message now only happens when the related UpdateAddHTLC is locked-in.](https://github.com/lightningnetwork/lnd/pull/6246)