From f2160273b7477c9f0202250dd803afb5da261813 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Mon, 11 Mar 2019 17:41:01 -0700 Subject: [PATCH] test: add new series of itests for various SCB restore scenarios In this commit, we add 4 new itests for exercising the SCB restore process via 4 primary scenarios: recover from backup using RPC, recover from file using RPC, recover channels during init/creation, recover channels during unlock. With all fields populated there're a total of 24 new scenarios to cover. At the time of authoring of this commit, the other scenarios (bits are: initiator, updates, private) have been left out for now, as they increased the run time of the integration tests significantly. --- lnd_test.go | 405 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 403 insertions(+), 2 deletions(-) diff --git a/lnd_test.go b/lnd_test.go index ed28f360d..80b8abb11 100644 --- a/lnd_test.go +++ b/lnd_test.go @@ -13190,7 +13190,7 @@ func testChannelBackupUpdates(net *lntest.NetworkHarness, t *harnessTest) { chanPoints = append(chanPoints, chanPoint) } - // Using this helper function, we'll maintain a pointe rot he latest + // Using this helper function, we'll maintain a pointer to the latest // channel backup so we can compare it to the on disk state. var currentBackup *lnrpc.ChanBackupSnapshot assertBackupNtfns := func(numNtfns int) { @@ -13333,7 +13333,10 @@ func testExportChannelBackup(net *lntest.NetworkHarness, t *harnessTest) { // backups of each of the channels. for _, chanPoint := range chanPoints { ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) - chanBackup, err := carol.ExportChannelBackup(ctxt, chanPoint) + req := &lnrpc.ExportChannelBackupRequest{ + ChanPoint: chanPoint, + } + chanBackup, err := carol.ExportChannelBackup(ctxt, req) if err != nil { t.Fatalf("unable to fetch backup for channel %v: %v", chanPoint, err) @@ -13444,6 +13447,401 @@ func testExportChannelBackup(net *lntest.NetworkHarness, t *harnessTest) { assertNumSingleBackups(0) assertMultiBackupFound()(false, nil) } + +// nodeRestorer is a function closure that allows each chanRestoreTestCase to +// control exactly *how* the prior node is restored. This might be using an +// backup obtained over RPC, or the file system, etc. +type nodeRestorer func() (*lntest.HarnessNode, error) + +// chanRestoreTestCase describes a test case for an end to end SCB restoration +// work flow. One node will start from scratch using an existing SCB. At the +// end of the est, both nodes should be made whole via the DLP protocol. +type chanRestoreTestCase struct { + // name is the name of the target test case. + name string + + // channelsUpdated is false then this means that no updates + // have taken place within the channel before restore. + // Otherwise, HTLCs will be settled between the two parties + // before restoration modifying the balance beyond the initial + // allocation. + channelsUpdated bool + + // initiator signals if Dave should be the one that opens the + // channel to Alice, or if it should be the other way around. + initiator bool + + // private signals if the channel from Dave to Carol should be + // private or not. + private bool + + // restoreMethod takes an old node, then returns a function + // closure that'll return the same node, but with its state + // restored via a custom method. We use this to abstract away + // _how_ a node is restored from our assertions once the node + // has been fully restored itself. + restoreMethod func(oldNode *lntest.HarnessNode, + backupFilePath string, + mnemonic []string) (nodeRestorer, error) +} + +// testChanRestoreScenario executes a chanRestoreTestCase from end to end, +// ensuring that after Dave restores his channel state according to the +// testCase, the DLP protocol is executed properly and both nodes are made +// whole. +func testChanRestoreScenario(t *harnessTest, net *lntest.NetworkHarness, + testCase *chanRestoreTestCase, password []byte) { + + const ( + chanAmt = btcutil.Amount(10000000) + pushAmt = btcutil.Amount(5000000) + ) + + ctxb := context.Background() + + // First, we'll create a brand new node we'll use within the test. If + // we have a custom backup file specified, then we'll also create that + // for use. + dave, mnemonic, err := net.NewNodeWithSeed( + "dave", nil, password, + ) + if err != nil { + t.Fatalf("unable to create new node: %v", err) + } + defer shutdownAndAssert(net, t, dave) + carol, err := net.NewNode("carol", nil) + if err != nil { + t.Fatalf("unable to make new node: %v", err) + } + defer shutdownAndAssert(net, t, carol) + + // Now that our new node is created, we'll give him some coins it can + // use to open channels with Carol. + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + err = net.SendCoins(ctxt, btcutil.SatoshiPerBitcoin, dave) + if err != nil { + t.Fatalf("unable to send coins to dave: %v", err) + } + + var from, to *lntest.HarnessNode + if testCase.initiator { + from, to = dave, carol + } else { + from, to = carol, dave + } + + // Next, we'll connect Dave to Carol, and open a new channel to her + // with a portion pushed. + if err := net.ConnectNodes(ctxt, dave, carol); err != nil { + t.Fatalf("unable to connect dave to carol: %v", err) + } + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPoint := openChannelAndAssert( + ctxt, t, net, from, to, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + Private: testCase.private, + }, + ) + + // Wait for both sides to see the opened channel. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = dave.WaitForNetworkChannelOpen(ctxt, chanPoint) + if err != nil { + t.Fatalf("dave didn't report channel: %v", err) + } + err = carol.WaitForNetworkChannelOpen(ctxt, chanPoint) + if err != nil { + t.Fatalf("carol didn't report channel: %v", err) + } + + // If both parties should start with existing channel updates, then + // we'll send+settle an HTLC between 'from' and 'to' now. + if testCase.channelsUpdated { + invoice := &lnrpc.Invoice{ + Memo: "testing", + Value: 10000, + } + invoiceResp, err := to.AddInvoice(ctxt, invoice) + if err != nil { + t.Fatalf("unable to add invoice: %v", err) + } + + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + err = completePaymentRequests( + ctxt, from, []string{invoiceResp.PaymentRequest}, + true, + ) + if err != nil { + t.Fatalf("unable to complete payments: %v", err) + } + } + + // Before we start the recovery, we'll record the balances of both + // Carol and Dave to ensure they both sweep their coins at the end. + balReq := &lnrpc.WalletBalanceRequest{} + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + carolBalResp, err := carol.WalletBalance(ctxt, balReq) + if err != nil { + t.Fatalf("unable to get carol's balance: %v", err) + } + carolStartingBalance := carolBalResp.ConfirmedBalance + + daveBalance, err := dave.WalletBalance(ctxt, balReq) + if err != nil { + t.Fatalf("unable to get carol's balance: %v", err) + } + daveStartingBalance := daveBalance.ConfirmedBalance + + // At this point, we'll now execute the restore method to give us the + // new node we should attempt our assertions against. + backupFilePath := dave.ChanBackupPath() + restoredNodeFunc, err := testCase.restoreMethod( + dave, backupFilePath, mnemonic, + ) + if err != nil { + t.Fatalf("unable to prep node restoration: %v", err) + } + + // TODO(roasbeef): assert recovery state in channel + + // Now that we're able to make our restored now, we'll shutdown the old + // Dave node as we'll be storing it shortly below. + shutdownAndAssert(net, t, dave) + + // Next, we'll make a new Dave and start the bulk of our recovery + // workflow. + dave, err = restoredNodeFunc() + if err != nil { + t.Fatalf("unable to restore node: %v", err) + } + + // Now that we have our new node up, we expect that it'll re-connect to + // Carol automatically based on the restored backup. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = net.EnsureConnected(ctxt, dave, carol) + if err != nil { + t.Fatalf("node didn't connect after recovery: %v", err) + } + + // TODO(roasbeef): move dave restarts? + + // Now we'll assert that both sides properly execute the DLP protocol. + // We grab their balances now to ensure that they're made whole at the + // end of the protocol. + assertDLPExecuted( + net, t, carol, carolStartingBalance, dave, daveStartingBalance, + ) +} + +// chanRestoreViaRPC is a helper test method that returns a nodeRestorer +// instance which will restore the target node from a password+seed, then +// trigger a SCB restore using the RPC interface. +func chanRestoreViaRPC(net *lntest.NetworkHarness, + password []byte, mnemonic []string, + multi []byte) (nodeRestorer, error) { + + backup := &lnrpc.RestoreChanBackupRequest_MultiChanBackup{ + MultiChanBackup: multi, + } + + ctxb := context.Background() + + return func() (*lntest.HarnessNode, error) { + newNode, err := net.RestoreNodeWithSeed( + "dave", nil, password, mnemonic, 1000, nil, + ) + if err != nil { + return nil, fmt.Errorf("unable to "+ + "restore node: %v", err) + } + + _, err = newNode.RestoreChannelBackups( + ctxb, &lnrpc.RestoreChanBackupRequest{ + Backup: backup, + }, + ) + if err != nil { + return nil, fmt.Errorf("unable "+ + "to restore backups: %v", err) + } + + return newNode, nil + }, nil +} + +// testChannelBackupRestore tests that we're able to recover from, and initiate +// the DLP protocol via: the RPC restore command, restoring on unlock, and +// restoring from initial wallet creation. We'll also alternate between +// restoring form the on disk file, and restoring from the exported RPC command +// as well. +func testChannelBackupRestore(net *lntest.NetworkHarness, t *harnessTest) { + password := []byte("El Psy Kongroo") + + ctxb := context.Background() + + var testCases = []chanRestoreTestCase{ + // Restore from backups obtained via the RPC interface. Dave + // was the initiator, of the non-advertised channel. + { + name: "restore from RPC backup", + channelsUpdated: false, + initiator: true, + private: false, + restoreMethod: func(oldNode *lntest.HarnessNode, + backupFilePath string, + mnemonic []string) (nodeRestorer, error) { + + // For this restoration method, we'll grab the + // current multi-channel backup from the old + // node, and use it to restore a new node + // within the closure. + req := &lnrpc.ChanBackupExportRequest{} + chanBackup, err := oldNode.ExportAllChannelBackups( + ctxb, req, + ) + if err != nil { + return nil, fmt.Errorf("unable to obtain "+ + "channel backup: %v", err) + } + + multi := chanBackup.MultiChanBackup.MultiChanBackup + + // In our nodeRestorer function, we'll restore + // the node from seed, then manually recover + // the channel backup. + return chanRestoreViaRPC( + net, password, mnemonic, multi, + ) + }, + }, + + // Restore the backup from the on-disk file, using the RPC + // interface. + { + name: "restore from backup file", + initiator: true, + private: false, + restoreMethod: func(oldNode *lntest.HarnessNode, + backupFilePath string, + mnemonic []string) (nodeRestorer, error) { + + // Read the entire Multi backup stored within + // this node's chaannels.backup file. + multi, err := ioutil.ReadFile(backupFilePath) + if err != nil { + return nil, err + } + + // Now that we have Dave's backup file, we'll + // create a new nodeRestorer that will restore + // using the on-disk channels.backup. + return chanRestoreViaRPC( + net, password, mnemonic, multi, + ) + }, + }, + + // Restore the backup as part of node initialization with the + // prior mnemonic and new backup seed. + { + name: "restore during creation", + initiator: true, + private: false, + restoreMethod: func(oldNode *lntest.HarnessNode, + backupFilePath string, + mnemonic []string) (nodeRestorer, error) { + + // First, fetch the current backup state as is, + // to obtain our latest Multi. + chanBackup, err := oldNode.ExportAllChannelBackups( + ctxb, &lnrpc.ChanBackupExportRequest{}, + ) + if err != nil { + return nil, fmt.Errorf("unable to obtain "+ + "channel backup: %v", err) + } + backupSnapshot := &lnrpc.ChanBackupSnapshot{ + MultiChanBackup: chanBackup.MultiChanBackup, + } + + // Create a new nodeRestorer that will restore + // the node using the Multi backup we just + // obtained above. + return func() (*lntest.HarnessNode, error) { + return net.RestoreNodeWithSeed( + "dave", nil, password, + mnemonic, 1000, backupSnapshot, + ) + }, nil + }, + }, + + // Restore the backup once the node has already been + // re-created, using the Unlock call. + { + name: "restore during unlock", + initiator: true, + private: false, + restoreMethod: func(oldNode *lntest.HarnessNode, + backupFilePath string, + mnemonic []string) (nodeRestorer, error) { + + // First, fetch the current backup state as is, + // to obtain our latest Multi. + chanBackup, err := oldNode.ExportAllChannelBackups( + ctxb, &lnrpc.ChanBackupExportRequest{}, + ) + if err != nil { + return nil, fmt.Errorf("unable to obtain "+ + "channel backup: %v", err) + } + backupSnapshot := &lnrpc.ChanBackupSnapshot{ + MultiChanBackup: chanBackup.MultiChanBackup, + } + + // Create a new nodeRestorer that will restore + // the node with its seed, but no channel + // backup, shutdown this initialized node, then + // restart it again using Unlock. + return func() (*lntest.HarnessNode, error) { + newNode, err := net.RestoreNodeWithSeed( + "dave", nil, password, + mnemonic, 1000, nil, + ) + if err != nil { + return nil, err + } + + err = net.RestartNode( + newNode, nil, backupSnapshot, + ) + if err != nil { + return nil, err + } + + return newNode, nil + }, nil + }, + }, + } + + // TODO(roasbeef): online vs offline close? + + // TODO(roasbeef): need to re-trigger the on-disk file once the node + // ann is updated? + + for _, testCase := range testCases { + success := t.t.Run(testCase.name, func(_ *testing.T) { + testChanRestoreScenario(t, net, &testCase, password) + }) + if !success { + break + } + } +} + type testCase struct { name string test func(net *lntest.NetworkHarness, t *harnessTest) @@ -13680,6 +14078,9 @@ var testsCases = []*testCase{ test: testExportChannelBackup, }, { + name: "channel backup restore", + test: testChannelBackupRestore, + }, } // TestLightningNetworkDaemon performs a series of integration tests amongst a