Merge pull request #8273 from guggero/bitcoind-26

GitHub: use bitcoind v26.0 for CI

lntest/bitcoind_common.go

@@ -15,6 +15,7 @@ import (
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/lightningnetwork/lnd/lntest/node"
+	"github.com/lightningnetwork/lnd/lntest/port"
 )
 
 // logDirPattern is the pattern of the name of the temporary log directory.
@@ -111,11 +112,10 @@ func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string,
 	}
 
 	zmqBlockAddr := fmt.Sprintf("tcp://127.0.0.1:%d",
-		node.NextAvailablePort())
-	zmqTxAddr := fmt.Sprintf("tcp://127.0.0.1:%d",
-		node.NextAvailablePort())
-	rpcPort := node.NextAvailablePort()
-	p2pPort := node.NextAvailablePort()
+		port.NextAvailablePort())
+	zmqTxAddr := fmt.Sprintf("tcp://127.0.0.1:%d", port.NextAvailablePort())
+	rpcPort := port.NextAvailablePort()
+	p2pPort := port.NextAvailablePort()
 
 	cmdArgs := []string{
 		"-datadir=" + tempBitcoindDir,

lntest/fee_service.go

@@ -10,7 +10,7 @@ import (
 	"testing"
 
 	"github.com/lightningnetwork/lnd"
-	"github.com/lightningnetwork/lnd/lntest/node"
+	"github.com/lightningnetwork/lnd/lntest/port"
 	"github.com/lightningnetwork/lnd/lnwallet/chainfee"
 	"github.com/stretchr/testify/require"
 )
@@ -60,11 +60,11 @@ type FeeService struct {
 // Compile-time check for the WebFeeService interface.
 var _ WebFeeService = (*FeeService)(nil)
 
-// Start spins up a go-routine to serve fee estimates.
+// NewFeeService spins up a go-routine to serve fee estimates.
 func NewFeeService(t *testing.T) *FeeService {
 	t.Helper()
 
-	port := node.NextAvailablePort()
+	port := port.NextAvailablePort()
 	f := FeeService{
 		T: t,
 		url: fmt.Sprintf(

lntest/harness_setup.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/btcsuite/btcd/integration/rpctest"
 	"github.com/lightningnetwork/lnd/lntest/node"
+	"github.com/lightningnetwork/lnd/lntest/wait"
 	"github.com/stretchr/testify/require"
 )
 
@@ -41,8 +42,15 @@ func SetupHarness(t *testing.T, binaryPath, dbBackendName string,
 	ht.stopChainBackend = cleanUp
 
 	// Connect our chainBackend to our miner.
-	t.Log("Connecting the miner with the chain backend...")
-	require.NoError(t, chainBackend.ConnectMiner(), "connect miner")
+	t.Logf("Connecting the miner at %v with the chain backend...",
+		miner.P2PAddress())
+
+	// Give the chain backend some time to fully start up, re-trying if any
+	// errors in connecting to the miner are encountered.
+	err := wait.NoError(func() error {
+		return chainBackend.ConnectMiner()
+	}, DefaultTimeout)
+	require.NoError(t, err, "connect miner")
 
 	// Start the HarnessTest with the chainBackend and miner.
 	ht.Start(chainBackend, miner)

lntest/node/config.go

@@ -4,16 +4,16 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"net"
 	"os"
 	"path"
 	"path/filepath"
-	"sync/atomic"
 
 	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/integration/rpctest"
 	"github.com/lightningnetwork/lnd"
 	"github.com/lightningnetwork/lnd/chanbackup"
 	"github.com/lightningnetwork/lnd/kvdb/etcd"
+	"github.com/lightningnetwork/lnd/lntest/port"
 	"github.com/lightningnetwork/lnd/lntest/wait"
 )
 
@@ -25,18 +25,9 @@ const (
 	// DefaultCSV is the CSV delay (remotedelay) we will start our test
 	// nodes with.
 	DefaultCSV = 4
-
-	// defaultNodePort is the start of the range for listening ports of
-	// harness nodes. Ports are monotonically increasing starting from this
-	// number and are determined by the results of NextAvailablePort().
-	defaultNodePort = 5555
 )
 
 var (
-	// lastPort is the last port determined to be free for use by a new
-	// node. It should be used atomically.
-	lastPort uint32 = defaultNodePort
-
 	// logOutput is a flag that can be set to append the output from the
 	// seed nodes to log files.
 	logOutput = flag.Bool("logoutput", false,
@@ -171,16 +162,16 @@ func (cfg BaseNodeConfig) ChanBackupPath() string {
 // current lightning network test.
 func (cfg *BaseNodeConfig) GenerateListeningPorts() {
 	if cfg.P2PPort == 0 {
-		cfg.P2PPort = NextAvailablePort()
+		cfg.P2PPort = port.NextAvailablePort()
 	}
 	if cfg.RPCPort == 0 {
-		cfg.RPCPort = NextAvailablePort()
+		cfg.RPCPort = port.NextAvailablePort()
 	}
 	if cfg.RESTPort == 0 {
-		cfg.RESTPort = NextAvailablePort()
+		cfg.RESTPort = port.NextAvailablePort()
 	}
 	if cfg.ProfilePort == 0 {
-		cfg.ProfilePort = NextAvailablePort()
+		cfg.ProfilePort = port.NextAvailablePort()
 	}
 }
 
@@ -259,13 +250,13 @@ func (cfg *BaseNodeConfig) GenArgs() []string {
 		args = append(
 			args, fmt.Sprintf(
 				"--db.etcd.embedded_client_port=%v",
-				NextAvailablePort(),
+				port.NextAvailablePort(),
 			),
 		)
 		args = append(
 			args, fmt.Sprintf(
 				"--db.etcd.embedded_peer_port=%v",
-				NextAvailablePort(),
+				port.NextAvailablePort(),
 			),
 		)
 		args = append(
@@ -333,34 +324,6 @@ func ExtraArgsEtcd(etcdCfg *etcd.Config, name string, cluster bool,
 	return extraArgs
 }
 
-// NextAvailablePort returns the first port that is available for listening by
-// a new node. It panics if no port is found and the maximum available TCP port
-// is reached.
-func NextAvailablePort() int {
-	port := atomic.AddUint32(&lastPort, 1)
-	for port < 65535 {
-		// If there are no errors while attempting to listen on this
-		// port, close the socket and return it as available. While it
-		// could be the case that some other process picks up this port
-		// between the time the socket is closed and it's reopened in
-		// the harness node, in practice in CI servers this seems much
-		// less likely than simply some other process already being
-		// bound at the start of the tests.
-		addr := fmt.Sprintf(ListenerFormat, port)
-		l, err := net.Listen("tcp4", addr)
-		if err == nil {
-			err := l.Close()
-			if err == nil {
-				return int(port)
-			}
-		}
-		port = atomic.AddUint32(&lastPort, 1)
-	}
-
-	// No ports available? Must be a mistake.
-	panic("no ports available for listening")
-}
-
 // GetLogDir returns the passed --logdir flag or the default value if it wasn't
 // set.
 func GetLogDir() string {
@@ -402,16 +365,10 @@ func GetBtcdBinary() string {
 	return ""
 }
 
-// GenerateBtcdListenerAddresses is a function that returns two listener
-// addresses with unique ports and should be used to overwrite rpctest's
-// default generator which is prone to use colliding ports.
-func GenerateBtcdListenerAddresses() (string, string) {
-	return fmt.Sprintf(ListenerFormat, NextAvailablePort()),
-		fmt.Sprintf(ListenerFormat, NextAvailablePort())
-}
-
-// ApplyPortOffset adds the given offset to the lastPort variable, making it
-// possible to run the tests in parallel without colliding on the same ports.
-func ApplyPortOffset(offset uint32) {
-	_ = atomic.AddUint32(&lastPort, offset)
+func init() {
+	// Before we start any node, we need to make sure that any btcd or
+	// bitcoind node that is started through the RPC harness uses a unique
+	// port as well to avoid any port collisions.
+	rpctest.ListenAddressGenerator =
+		port.GenerateSystemUniqueListenerAddresses
 }

lntest/port/port.go

@@ -0,0 +1,150 @@
+package port
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"time"
+)
+
+const (
+	// ListenerFormat is the format string that is used to generate local
+	// listener addresses.
+	ListenerFormat = "127.0.0.1:%d"
+
+	// defaultNodePort is the start of the range for listening ports of
+	// harness nodes. Ports are monotonically increasing starting from this
+	// number and are determined by the results of NextAvailablePort().
+	defaultNodePort int = 10000
+
+	// uniquePortFile is the name of the file that is used to store the
+	// last port that was used by a node. This is used to make sure that
+	// the same port is not used by multiple nodes at the same time. The
+	// file is located in the temp directory of a system.
+	uniquePortFile = "rpctest-port"
+)
+
+var (
+	// portFileMutex is a mutex that is used to make sure that the port file
+	// is not accessed by multiple goroutines of the same process at the
+	// same time. This is used in conjunction with the lock file to make
+	// sure that the port file is not accessed by multiple processes at the
+	// same time either. So the lock file is to guard between processes and
+	// the mutex is to guard between goroutines of the same process.
+	portFileMutex sync.Mutex
+)
+
+// NextAvailablePort returns the first port that is available for listening by a
+// new node, using a lock file to make sure concurrent access for parallel tasks
+// on the same system don't re-use the same port.
+func NextAvailablePort() int {
+	portFileMutex.Lock()
+	defer portFileMutex.Unlock()
+
+	lockFile := filepath.Join(os.TempDir(), uniquePortFile+".lock")
+	timeout := time.After(time.Second)
+
+	var (
+		lockFileHandle *os.File
+		err            error
+	)
+	for {
+		// Attempt to acquire the lock file. If it already exists, wait
+		// for a bit and retry.
+		lockFileHandle, err = os.OpenFile(
+			lockFile, os.O_CREATE|os.O_EXCL, 0600,
+		)
+		if err == nil {
+			// Lock acquired.
+			break
+		}
+
+		// Wait for a bit and retry.
+		select {
+		case <-timeout:
+			panic("timeout waiting for lock file")
+		case <-time.After(10 * time.Millisecond):
+		}
+	}
+
+	// Release the lock file when we're done.
+	defer func() {
+		// Always close file first, Windows won't allow us to remove it
+		// otherwise.
+		_ = lockFileHandle.Close()
+		err := os.Remove(lockFile)
+		if err != nil {
+			panic(fmt.Errorf("couldn't remove lock file: %w", err))
+		}
+	}()
+
+	portFile := filepath.Join(os.TempDir(), uniquePortFile)
+	port, err := os.ReadFile(portFile)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			panic(fmt.Errorf("error reading port file: %w", err))
+		}
+		port = []byte(strconv.Itoa(defaultNodePort))
+	}
+
+	lastPort, err := strconv.Atoi(string(port))
+	if err != nil {
+		panic(fmt.Errorf("error parsing port: %w", err))
+	}
+
+	// We take the next one.
+	lastPort++
+	for lastPort < 65535 {
+		// If there are no errors while attempting to listen on this
+		// port, close the socket and return it as available. While it
+		// could be the case that some other process picks up this port
+		// between the time the socket is closed, and it's reopened in
+		// the harness node, in practice in CI servers this seems much
+		// less likely than simply some other process already being
+		// bound at the start of the tests.
+		addr := fmt.Sprintf(ListenerFormat, lastPort)
+		l, err := net.Listen("tcp4", addr)
+		if err == nil {
+			err := l.Close()
+			if err == nil {
+				err := os.WriteFile(
+					portFile,
+					[]byte(strconv.Itoa(lastPort)), 0600,
+				)
+				if err != nil {
+					panic(fmt.Errorf("error updating "+
+						"port file: %w", err))
+				}
+
+				return lastPort
+			}
+		}
+		lastPort++
+
+		// Start from the beginning if we reached the end of the port
+		// range. We need to do this because the lock file now is
+		// persistent across runs on the same machine during the same
+		// boot/uptime cycle. So in order to make this work on
+		// developer's machines, we need to reset the port to the
+		// default value when we reach the end of the range.
+		if lastPort == 65535 {
+			lastPort = defaultNodePort
+		}
+	}
+
+	// No ports available? Must be a mistake.
+	panic("no ports available for listening")
+}
+
+// GenerateSystemUniqueListenerAddresses is a function that returns two
+// listener addresses with unique ports per system and should be used to
+// overwrite rpctest's default generator which is prone to use colliding ports.
+func GenerateSystemUniqueListenerAddresses() (string, string) {
+	port1 := NextAvailablePort()
+	port2 := NextAvailablePort()
+	return fmt.Sprintf(ListenerFormat, port1),
+		fmt.Sprintf(ListenerFormat, port2)
+}

lntest/unittest/backend.go

@@ -0,0 +1,207 @@
+package unittest
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/integration/rpctest"
+	"github.com/btcsuite/btcwallet/chain"
+	"github.com/btcsuite/btcwallet/walletdb"
+	"github.com/lightninglabs/neutrino"
+	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/lightningnetwork/lnd/lntest/port"
+	"github.com/lightningnetwork/lnd/lntest/wait"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	// TrickleInterval is the interval at which the miner should trickle
+	// transactions to its peers. We'll set it small to ensure the miner
+	// propagates transactions quickly in the tests.
+	TrickleInterval = 10 * time.Millisecond
+)
+
+var (
+	// NetParams are the default network parameters for the tests.
+	NetParams = &chaincfg.RegressionNetParams
+)
+
+// NewMiner spawns testing harness backed by a btcd node that can serve as a
+// miner.
+func NewMiner(t *testing.T, netParams *chaincfg.Params, extraArgs []string,
+	createChain bool, spendableOutputs uint32) *rpctest.Harness {
+
+	t.Helper()
+
+	// Add the trickle interval argument to the extra args.
+	trickle := fmt.Sprintf("--trickleinterval=%v", TrickleInterval)
+	extraArgs = append(extraArgs, trickle)
+
+	node, err := rpctest.New(netParams, nil, extraArgs, "")
+	require.NoError(t, err, "unable to create backend node")
+	t.Cleanup(func() {
+		require.NoError(t, node.TearDown())
+	})
+
+	// We want to overwrite some of the connection settings to make the
+	// tests more robust. We might need to restart the backend while there
+	// are already blocks present, which will take a bit longer than the
+	// 1 second the default settings amount to. Doubling both values will
+	// give us retries up to 4 seconds.
+	node.MaxConnRetries = rpctest.DefaultMaxConnectionRetries * 2
+	node.ConnectionRetryTimeout = rpctest.DefaultConnectionRetryTimeout * 2
+
+	if err := node.SetUp(createChain, spendableOutputs); err != nil {
+		t.Fatalf("unable to set up backend node: %v", err)
+	}
+
+	return node
+}
+
+// NewBitcoindBackend spawns a new bitcoind node that connects to a miner at the
+// specified address. The txindex boolean can be set to determine whether the
+// backend node should maintain a transaction index. The rpcpolling boolean
+// can be set to determine whether bitcoind's RPC polling interface should be
+// used for block and tx notifications or if its ZMQ interface should be used.
+// A connection to the newly spawned bitcoind node is returned.
+func NewBitcoindBackend(t *testing.T, netParams *chaincfg.Params,
+	minerAddr string, txindex, rpcpolling bool) *chain.BitcoindConn {
+
+	t.Helper()
+
+	tempBitcoindDir := t.TempDir()
+
+	rpcPort := port.NextAvailablePort()
+	zmqBlockPort := port.NextAvailablePort()
+	zmqTxPort := port.NextAvailablePort()
+	zmqBlockHost := fmt.Sprintf("tcp://127.0.0.1:%d", zmqBlockPort)
+	zmqTxHost := fmt.Sprintf("tcp://127.0.0.1:%d", zmqTxPort)
+
+	args := []string{
+		"-connect=" + minerAddr,
+		"-datadir=" + tempBitcoindDir,
+		"-regtest",
+		"-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6fd$507c670e800a95" +
+			"284294edb5773b05544b220110063096c221be9933c82d38e1",
+		fmt.Sprintf("-rpcport=%d", rpcPort),
+		"-disablewallet",
+		"-zmqpubrawblock=" + zmqBlockHost,
+		"-zmqpubrawtx=" + zmqTxHost,
+	}
+	if txindex {
+		args = append(args, "-txindex")
+	}
+
+	bitcoind := exec.Command("bitcoind", args...)
+	if err := bitcoind.Start(); err != nil {
+		t.Fatalf("unable to start bitcoind: %v", err)
+	}
+	t.Cleanup(func() {
+		_ = bitcoind.Process.Kill()
+		_ = bitcoind.Wait()
+	})
+
+	// Wait for the bitcoind instance to start up.
+	time.Sleep(time.Second)
+
+	host := fmt.Sprintf("127.0.0.1:%d", rpcPort)
+	cfg := &chain.BitcoindConfig{
+		ChainParams: netParams,
+		Host:        host,
+		User:        "weks",
+		Pass:        "weks",
+		// Fields only required for pruned nodes, not needed for these
+		// tests.
+		Dialer:             nil,
+		PrunedModeMaxPeers: 0,
+	}
+
+	if rpcpolling {
+		cfg.PollingConfig = &chain.PollingConfig{
+			BlockPollingInterval: time.Millisecond * 20,
+			TxPollingInterval:    time.Millisecond * 20,
+		}
+	} else {
+		cfg.ZMQConfig = &chain.ZMQConfig{
+			ZMQBlockHost:    zmqBlockHost,
+			ZMQTxHost:       zmqTxHost,
+			ZMQReadDeadline: 5 * time.Second,
+		}
+	}
+
+	var conn *chain.BitcoindConn
+	err := wait.NoError(func() error {
+		var err error
+		conn, err = chain.NewBitcoindConn(cfg)
+		if err != nil {
+			return err
+		}
+
+		return conn.Start()
+	}, 10*time.Second)
+	if err != nil {
+		t.Fatalf("unable to establish connection to bitcoind at %v: "+
+			"%v", tempBitcoindDir, err)
+	}
+	t.Cleanup(conn.Stop)
+
+	return conn
+}
+
+// NewNeutrinoBackend spawns a new neutrino node that connects to a miner at
+// the specified address.
+func NewNeutrinoBackend(t *testing.T, netParams *chaincfg.Params,
+	minerAddr string) *neutrino.ChainService {
+
+	t.Helper()
+
+	spvDir := t.TempDir()
+
+	dbName := filepath.Join(spvDir, "neutrino.db")
+	spvDatabase, err := walletdb.Create(
+		"bdb", dbName, true, kvdb.DefaultDBTimeout,
+	)
+	if err != nil {
+		t.Fatalf("unable to create walletdb: %v", err)
+	}
+	t.Cleanup(func() {
+		spvDatabase.Close()
+	})
+
+	// Create an instance of neutrino connected to the running btcd
+	// instance.
+	spvConfig := neutrino.Config{
+		DataDir:      spvDir,
+		Database:     spvDatabase,
+		ChainParams:  *netParams,
+		ConnectPeers: []string{minerAddr},
+	}
+	spvNode, err := neutrino.NewChainService(spvConfig)
+	if err != nil {
+		t.Fatalf("unable to create neutrino: %v", err)
+	}
+
+	// We'll also wait for the instance to sync up fully to the chain
+	// generated by the btcd instance.
+	_ = spvNode.Start()
+	for !spvNode.IsCurrent() {
+		time.Sleep(time.Millisecond * 100)
+	}
+	t.Cleanup(func() {
+		_ = spvNode.Stop()
+	})
+
+	return spvNode
+}
+
+func init() {
+	// Before we start any node, we need to make sure that any btcd or
+	// bitcoind node that is started through the RPC harness uses a unique
+	// port as well to avoid any port collisions.
+	rpctest.ListenAddressGenerator =
+		port.GenerateSystemUniqueListenerAddresses
+}