From 29667730188bd320cd037e9aa757d765c2362984 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Thu, 28 Jul 2022 17:36:56 +0800 Subject: [PATCH] lntemp: add more assertions and supporting methods --- lntemp/harness.go | 154 +++++++++++++++++++++++-- lntemp/harness_assertion.go | 97 ++++++++++++++-- lntemp/harness_miner.go | 26 +++++ lntemp/rpc/lnd.go | 44 +++++++ lntemp/rpc/wallet_kit.go | 26 +++++ lntest/itest/lnd_channel_graph_test.go | 2 +- lntest/itest/lnd_funding_test.go | 41 ++++--- lntest/itest/temp_lnd_test.go | 1 + 8 files changed, 355 insertions(+), 36 deletions(-) diff --git a/lntemp/harness.go b/lntemp/harness.go index bbcac60a5..7638a1674 100644 --- a/lntemp/harness.go +++ b/lntemp/harness.go @@ -649,6 +649,15 @@ type OpenChannelParams struct { // CommitmentType is the commitment type that should be used for the // channel to be opened. CommitmentType lnrpc.CommitmentType + + // ZeroConf is used to determine if the channel will be a zero-conf + // channel. This only works if the explicit negotiation is used with + // anchors or script enforced leases. + ZeroConf bool + + // ScidAlias denotes whether the channel will be an option-scid-alias + // channel type negotiation. + ScidAlias bool } // OpenChannelAssertPending attempts to open a channel between srcNode and @@ -679,6 +688,8 @@ func (h *HarnessTest) OpenChannelAssertPending( FundingShim: p.FundingShim, SatPerByte: int64(p.SatPerVByte), CommitmentType: p.CommitmentType, + ZeroConf: p.ZeroConf, + ScidAlias: p.ScidAlias, } respStream := srcNode.RPC.OpenChannel(openReq) @@ -742,11 +753,13 @@ func (h *HarnessTest) OpenChannel(alice, bob *node.HarnessNode, return fundingChanPoint } -// closeChannel attempts to close the channel indicated by the passed channel -// point, initiated by the passed node. Once the CloseChannel rpc is called, it -// will consume one event and assert it's a close pending event. In addition, -// it will check that the closing tx can be found in the mempool. -func (h *HarnessTest) closeChannel(hn *node.HarnessNode, cp *lnrpc.ChannelPoint, +// CloseChannelAssertPending attempts to close the channel indicated by the +// passed channel point, initiated by the passed node. Once the CloseChannel +// rpc is called, it will consume one event and assert it's a close pending +// event. In addition, it will check that the closing tx can be found in the +// mempool. +func (h *HarnessTest) CloseChannelAssertPending(hn *node.HarnessNode, + cp *lnrpc.ChannelPoint, force bool) (rpc.CloseChanClient, *chainhash.Hash) { // Calls the rpc to close the channel. @@ -776,7 +789,7 @@ func (h *HarnessTest) closeChannel(hn *node.HarnessNode, cp *lnrpc.ChannelPoint, return stream, closeTxid } -// CloseChannel attempts to close a non-anchored channel identified by the +// CloseChannel attempts to coop close a non-anchored channel identified by the // passed channel point owned by the passed harness node. The following items // are asserted, // 1. a close pending event is sent from the close channel client. @@ -786,17 +799,41 @@ func (h *HarnessTest) closeChannel(hn *node.HarnessNode, cp *lnrpc.ChannelPoint, // 5. the node reports zero waiting close channels. // 6. the node receives a topology update regarding the channel close. func (h *HarnessTest) CloseChannel(hn *node.HarnessNode, - cp *lnrpc.ChannelPoint, force bool) *chainhash.Hash { + cp *lnrpc.ChannelPoint) *chainhash.Hash { - stream, _ := h.closeChannel(hn, cp, force) + stream, _ := h.CloseChannelAssertPending(hn, cp, false) - return h.assertChannelClosed(hn, cp, false, stream) + return h.assertChannelCoopClosed(hn, cp, false, stream) +} + +// ForceCloseChannel attempts to force close a non-anchored channel identified +// by the passed channel point owned by the passed harness node. The following +// items are asserted, +// 1. a close pending event is sent from the close channel client. +// 2. the closing tx is found in the mempool. +// 3. the node reports the channel being waiting to close. +// 4. a block is mined and the closing tx should be found in it. +// 5. the node reports zero waiting close channels. +// 6. the node receives a topology update regarding the channel close. +// 7. mine DefaultCSV-1 blocks. +// 8. the node reports zero pending force close channels. +func (h *HarnessTest) ForceCloseChannel(hn *node.HarnessNode, + cp *lnrpc.ChannelPoint) *chainhash.Hash { + + stream, _ := h.CloseChannelAssertPending(hn, cp, true) + + closingTxid := h.AssertStreamChannelForceClosed(hn, cp, false, stream) + + // Cleanup the force close. + h.CleanupForceClose(hn, cp) + + return closingTxid } // CloseChannelAssertErr closes the given channel and asserts an error // returned. func (h *HarnessTest) CloseChannelAssertErr(hn *node.HarnessNode, - cp *lnrpc.ChannelPoint, force bool) { + cp *lnrpc.ChannelPoint, force bool) error { // Calls the rpc to close the channel. closeReq := &lnrpc.CloseChannelRequest{ @@ -811,6 +848,8 @@ func (h *HarnessTest) CloseChannelAssertErr(hn *node.HarnessNode, _, err := h.ReceiveCloseChannelUpdate(stream) require.Errorf(h, err, "%s: expect close channel to return an error", hn.Name()) + + return err } // IsNeutrinoBackend returns a bool indicating whether the node is using a @@ -971,3 +1010,98 @@ func (h *HarnessTest) CompletePaymentRequestsNoWait(hn *node.HarnessNode, }, DefaultTimeout) require.NoError(h, err, "timeout while checking for channel updates") } + +// OpenChannelPsbt attempts to open a channel between srcNode and destNode with +// the passed channel funding parameters. It will assert if the expected step +// of funding the PSBT is not received from the source node. +func (h *HarnessTest) OpenChannelPsbt(srcNode, destNode *node.HarnessNode, + p OpenChannelParams) (rpc.OpenChanClient, []byte) { + + // Wait until srcNode and destNode have the latest chain synced. + // Otherwise, we may run into a check within the funding manager that + // prevents any funding workflows from being kicked off if the chain + // isn't yet synced. + h.WaitForBlockchainSync(srcNode) + h.WaitForBlockchainSync(destNode) + + // Send the request to open a channel to the source node now. This will + // open a long-lived stream where we'll receive status updates about + // the progress of the channel. + // respStream := h.OpenChannelStreamAndAssert(srcNode, destNode, p) + req := &lnrpc.OpenChannelRequest{ + NodePubkey: destNode.PubKey[:], + LocalFundingAmount: int64(p.Amt), + PushSat: int64(p.PushAmt), + Private: p.Private, + SpendUnconfirmed: p.SpendUnconfirmed, + MinHtlcMsat: int64(p.MinHtlc), + FundingShim: p.FundingShim, + } + respStream := srcNode.RPC.OpenChannel(req) + + // Consume the "PSBT funding ready" update. This waits until the node + // notifies us that the PSBT can now be funded. + resp := h.ReceiveOpenChannelUpdate(respStream) + upd, ok := resp.Update.(*lnrpc.OpenStatusUpdate_PsbtFund) + require.Truef(h, ok, "expected PSBT funding update, got %v", resp) + + return respStream, upd.PsbtFund.Psbt +} + +// CleanupForceClose mines a force close commitment found in the mempool and +// the following sweep transaction from the force closing node. +func (h *HarnessTest) CleanupForceClose(hn *node.HarnessNode, + chanPoint *lnrpc.ChannelPoint) { + + // Wait for the channel to be marked pending force close. + h.AssertNumPendingForceClose(hn, 1) + + // Mine enough blocks for the node to sweep its funds from the force + // closed channel. + // + // The commit sweep resolver is able to broadcast the sweep tx up to + // one block before the CSV elapses, so wait until defaulCSV-1. + h.Miner.MineBlocks(lntest.DefaultCSV - 1) + + // The node should now sweep the funds, clean up by mining the sweeping + // tx. + h.Miner.MineBlocksAndAssertNumTxes(1, 1) + + // Mine blocks to get any second level HTLC resolved. If there are no + // HTLCs, this will behave like h.AssertNumPendingCloseChannels. + h.mineTillForceCloseResolved(hn) +} + +// mineTillForceCloseResolved asserts that the number of pending close channels +// are zero. Each time it checks, a new block is mined using MineBlocksSlow to +// give the node some time to catch up the chain. +// +// NOTE: this method is a workaround to make sure we have a clean mempool at +// the end of a channel force closure. We cannot directly mine blocks and +// assert channels being fully closed because the subsystems in lnd don't share +// the same block height. This is especially the case when blocks are produced +// too fast. +// TODO(yy): remove this workaround when syncing blocks are unified in all the +// subsystems. +func (h *HarnessTest) mineTillForceCloseResolved(hn *node.HarnessNode) { + _, startHeight := h.Miner.GetBestBlock() + + err := wait.NoError(func() error { + resp := hn.RPC.PendingChannels() + total := len(resp.PendingForceClosingChannels) + if total != 0 { + h.Miner.MineBlocksSlow(1) + + return fmt.Errorf("expected num of pending force " + + "close channel to be zero") + } + + _, height := h.Miner.GetBestBlock() + h.Logf("Mined %d blocks while waiting for force closed "+ + "channel to be resolved", height-startHeight) + + return nil + }, DefaultTimeout) + + require.NoErrorf(h, err, "assert force close resolved timeout") +} diff --git a/lntemp/harness_assertion.go b/lntemp/harness_assertion.go index 381074160..33d6d04c7 100644 --- a/lntemp/harness_assertion.go +++ b/lntemp/harness_assertion.go @@ -398,12 +398,16 @@ func (h HarnessTest) WaitForChannelCloseEvent( // AssertNumWaitingClose checks that a PendingChannels response from the node // reports the expected number of waiting close channels. -func (h *HarnessTest) AssertNumWaitingClose(hn *node.HarnessNode, num int) { +func (h *HarnessTest) AssertNumWaitingClose(hn *node.HarnessNode, + num int) []*lnrpc.PendingChannelsResponse_WaitingCloseChannel { + + var channels []*lnrpc.PendingChannelsResponse_WaitingCloseChannel oldWaiting := hn.State.CloseChannel.WaitingClose err := wait.NoError(func() error { resp := hn.RPC.PendingChannels() - total := len(resp.WaitingCloseChannels) + channels = resp.WaitingCloseChannels + total := len(channels) got := total - oldWaiting if got == num { @@ -416,18 +420,22 @@ func (h *HarnessTest) AssertNumWaitingClose(hn *node.HarnessNode, num int) { require.NoErrorf(h, err, "%s: assert waiting close timeout", hn.Name()) + + return channels } // AssertNumPendingForceClose checks that a PendingChannels response from the // node reports the expected number of pending force close channels. func (h *HarnessTest) AssertNumPendingForceClose(hn *node.HarnessNode, - num int) { + num int) []*lnrpc.PendingChannelsResponse_ForceClosedChannel { + var channels []*lnrpc.PendingChannelsResponse_ForceClosedChannel oldForce := hn.State.CloseChannel.PendingForceClose err := wait.NoError(func() error { resp := hn.RPC.PendingChannels() - total := len(resp.PendingForceClosingChannels) + channels = resp.PendingForceClosingChannels + total := len(channels) got := total - oldForce if got == num { @@ -440,11 +448,13 @@ func (h *HarnessTest) AssertNumPendingForceClose(hn *node.HarnessNode, require.NoErrorf(h, err, "%s: assert pending force close timeout", hn.Name()) + + return channels } -// assertChannelClosed asserts that the channel is properly cleaned up after -// initiating a cooperative or local close. -func (h *HarnessTest) assertChannelClosed(hn *node.HarnessNode, +// assertChannelCoopClosed asserts that the channel is properly cleaned up +// after initiating a cooperative close. +func (h *HarnessTest) assertChannelCoopClosed(hn *node.HarnessNode, cp *lnrpc.ChannelPoint, anchors bool, stream rpc.CloseChanClient) *chainhash.Hash { @@ -480,6 +490,52 @@ func (h *HarnessTest) assertChannelClosed(hn *node.HarnessNode, return closingTxid } +// AssertStreamChannelForceClosed reads an update from the close channel client +// stream and asserts that the mempool state and node's topology match a local +// force close. In specific, +// - assert the channel is waiting close and has the expected ChanStatusFlags. +// - assert the mempool has the closing txes and anchor sweeps. +// - mine a block and assert the closing txid is mined. +// - assert the channel is pending force close. +// - assert the node has seen the channel close update. +func (h *HarnessTest) AssertStreamChannelForceClosed(hn *node.HarnessNode, + cp *lnrpc.ChannelPoint, anchors bool, + stream rpc.CloseChanClient) *chainhash.Hash { + + // Assert the channel is waiting close. + resp := h.AssertChannelWaitingClose(hn, cp) + + // Assert that the channel is in local force broadcasted. + require.Contains(h, resp.Channel.ChanStatusFlags, + channeldb.ChanStatusLocalCloseInitiator.String(), + "channel not coop broadcasted") + + // We'll now, generate a single block, wait for the final close status + // update, then ensure that the closing transaction was included in the + // block. If there are anchors, we also expect an anchor sweep. + expectedTxes := 1 + if anchors { + expectedTxes = 2 + } + block := h.Miner.MineBlocksAndAssertNumTxes(1, expectedTxes)[0] + + // Consume one close event and assert the closing txid can be found in + // the block. + closingTxid := h.WaitForChannelCloseEvent(stream) + h.Miner.AssertTxInBlock(block, closingTxid) + + // We should see zero waiting close channels and 1 pending force close + // channels now. + h.AssertNumWaitingClose(hn, 0) + h.AssertNumPendingForceClose(hn, 1) + + // Finally, check that the node's topology graph has seen this channel + // closed. + h.AssertTopologyChannelClosed(hn, cp) + + return closingTxid +} + // AssertChannelPolicyUpdate checks that the required policy update has // happened on the given node. func (h *HarnessTest) AssertChannelPolicyUpdate(hn *node.HarnessNode, @@ -822,3 +878,30 @@ func (h *HarnessTest) AssertInvoiceSettled(hn *node.HarnessNode, addr []byte) { }, DefaultTimeout) require.NoError(h, err, "timeout waiting for invoice settled state") } + +// AssertNodeNumChannels polls the provided node's list channels rpc until it +// reaches the desired number of total channels. +func (h *HarnessTest) AssertNodeNumChannels(hn *node.HarnessNode, + numChannels int) { + + // Get the total number of channels. + old := hn.State.OpenChannel.Active + hn.State.OpenChannel.Inactive + + err := wait.NoError(func() error { + // We require the RPC call to be succeeded and won't wait for + // it as it's an unexpected behavior. + chanInfo := hn.RPC.ListChannels(&lnrpc.ListChannelsRequest{}) + + // Return true if the query returned the expected number of + // channels. + num := len(chanInfo.Channels) - old + if num != numChannels { + return fmt.Errorf("expected %v channels, got %v", + numChannels, num) + } + + return nil + }, DefaultTimeout) + + require.NoError(h, err, "timeout checking node's num of channels") +} diff --git a/lntemp/harness_miner.go b/lntemp/harness_miner.go index fc6c8fa3f..8509d3e17 100644 --- a/lntemp/harness_miner.go +++ b/lntemp/harness_miner.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg" @@ -27,6 +28,8 @@ const ( // minerLogDir is the default log dir for the miner node. minerLogDir = ".minerlogs" + + slowMineDelay = 20 * time.Millisecond ) var harnessNetParams = &chaincfg.RegressionNetParams @@ -301,3 +304,26 @@ func (h *HarnessMiner) SendOutput(newOutput *wire.TxOut, return hash } + +// MineBlocksSlow mines 'num' of blocks. Between each mined block an artificial +// delay is introduced to give all network participants time to catch up. +func (h *HarnessMiner) MineBlocksSlow(num uint32) []*wire.MsgBlock { + blocks := make([]*wire.MsgBlock, num) + blockHashes := make([]*chainhash.Hash, 0, num) + + for i := uint32(0); i < num; i++ { + generatedHashes := h.GenerateBlocks(1) + blockHashes = append(blockHashes, generatedHashes...) + + time.Sleep(slowMineDelay) + } + + for i, blockHash := range blockHashes { + block, err := h.Client.GetBlock(blockHash) + require.NoError(h, err, "get blocks") + + blocks[i] = block + } + + return blocks +} diff --git a/lntemp/rpc/lnd.go b/lntemp/rpc/lnd.go index b25e92ee1..4c9f4cfd1 100644 --- a/lntemp/rpc/lnd.go +++ b/lntemp/rpc/lnd.go @@ -262,3 +262,47 @@ func (h *HarnessRPC) AbandonChannel( return resp } + +// ExportAllChanBackups makes a RPC call to the node's ExportAllChannelBackups +// and asserts. +func (h *HarnessRPC) ExportAllChanBackups() *lnrpc.ChanBackupSnapshot { + ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout) + defer cancel() + + req := &lnrpc.ChanBackupExportRequest{} + chanBackup, err := h.LN.ExportAllChannelBackups(ctxt, req) + h.NoError(err, "ExportAllChannelBackups") + + return chanBackup +} + +// ExportChanBackup makes a RPC call to the node's ExportChannelBackup +// and asserts. +func (h *HarnessRPC) ExportChanBackup( + chanPoint *lnrpc.ChannelPoint) *lnrpc.ChannelBackup { + + ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout) + defer cancel() + + req := &lnrpc.ExportChannelBackupRequest{ + ChanPoint: chanPoint, + } + chanBackup, err := h.LN.ExportChannelBackup(ctxt, req) + h.NoError(err, "ExportChannelBackup") + + return chanBackup +} + +// RestoreChanBackups makes a RPC call to the node's RestoreChannelBackups and +// asserts. +func (h *HarnessRPC) RestoreChanBackups( + req *lnrpc.RestoreChanBackupRequest) *lnrpc.RestoreBackupResponse { + + ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout) + defer cancel() + + resp, err := h.LN.RestoreChannelBackups(ctxt, req) + h.NoError(err, "RestoreChannelBackups") + + return resp +} diff --git a/lntemp/rpc/wallet_kit.go b/lntemp/rpc/wallet_kit.go index c68e38667..ed0f86960 100644 --- a/lntemp/rpc/wallet_kit.go +++ b/lntemp/rpc/wallet_kit.go @@ -47,3 +47,29 @@ func (h *HarnessRPC) SendOutputs( return resp } + +// FundPsbt makes a RPC call to node's FundPsbt and asserts. +func (h *HarnessRPC) FundPsbt( + req *walletrpc.FundPsbtRequest) *walletrpc.FundPsbtResponse { + + ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout) + defer cancel() + + resp, err := h.WalletKit.FundPsbt(ctxt, req) + h.NoError(err, "FundPsbt") + + return resp +} + +// FinalizePsbt makes a RPC call to node's FinalizePsbt and asserts. +func (h *HarnessRPC) FinalizePsbt( + req *walletrpc.FinalizePsbtRequest) *walletrpc.FinalizePsbtResponse { + + ctxt, cancel := context.WithTimeout(h.runCtx, DefaultTimeout) + defer cancel() + + resp, err := h.WalletKit.FinalizePsbt(ctxt, req) + h.NoError(err, "FinalizePsbt") + + return resp +} diff --git a/lntest/itest/lnd_channel_graph_test.go b/lntest/itest/lnd_channel_graph_test.go index 9449e12de..0010dbdb5 100644 --- a/lntest/itest/lnd_channel_graph_test.go +++ b/lntest/itest/lnd_channel_graph_test.go @@ -64,7 +64,7 @@ func testUpdateChanStatus(ht *lntemp.HarnessTest) { chanPoint := ht.OpenChannel( alice, bob, lntemp.OpenChannelParams{Amt: chanAmt}, ) - defer ht.CloseChannel(alice, chanPoint, false) + defer ht.CloseChannel(alice, chanPoint) // assertEdgeDisabled ensures that Alice has the correct Disabled state // for given channel from her DescribeGraph. diff --git a/lntest/itest/lnd_funding_test.go b/lntest/itest/lnd_funding_test.go index fa6133a2d..28c5b8320 100644 --- a/lntest/itest/lnd_funding_test.go +++ b/lntest/itest/lnd_funding_test.go @@ -69,27 +69,24 @@ func testBasicChannelFunding(ht *lntemp.HarnessTest) { require.Equal(ht, chansCommitType, daveChan.CommitmentType, "commit types don't match") - // Now check that the commitment type reported - // by both nodes is what we expect. It will be - // the minimum of the two nodes' preference, in - // the order Legacy, Tweakless, Anchors. + // Now check that the commitment type reported by both nodes is + // what we expect. It will be the minimum of the two nodes' + // preference, in the order Legacy, Tweakless, Anchors. expType := carolCommitType switch daveCommitType { - // Dave supports anchors, type will be what - // Carol supports. + // Dave supports anchors, type will be what Carol supports. case lnrpc.CommitmentType_ANCHORS: - // Dave only supports tweakless, channel will - // be downgraded to this type if Carol supports - // anchors. + // Dave only supports tweakless, channel will be downgraded to + // this type if Carol supports anchors. case lnrpc.CommitmentType_STATIC_REMOTE_KEY: if expType == lnrpc.CommitmentType_ANCHORS { expType = lnrpc.CommitmentType_STATIC_REMOTE_KEY } - // Dave only supoprts legacy type, channel will - // be downgraded to this type. + // Dave only supoprts legacy type, channel will be downgraded + // to this type. case lnrpc.CommitmentType_LEGACY: expType = lnrpc.CommitmentType_LEGACY @@ -97,8 +94,7 @@ func testBasicChannelFunding(ht *lntemp.HarnessTest) { ht.Fatalf("invalid commit type %v", daveCommitType) } - // Check that the signalled type matches what we - // expect. + // Check that the signalled type matches what we expect. switch { case expType == lnrpc.CommitmentType_ANCHORS && chansCommitType == lnrpc.CommitmentType_ANCHORS: @@ -115,8 +111,8 @@ func testBasicChannelFunding(ht *lntemp.HarnessTest) { "%v", expType, chansCommitType) } - // As we've concluded this sub-test case we'll - // now close out the channel for both sides. + // As we've concluded this sub-test case we'll now close out + // the channel for both sides. closeChan() } @@ -217,7 +213,7 @@ func basicChannelFundingTest(ht *lntemp.HarnessTest, // Finally, immediately close the channel. This function will // also block until the channel is closed and will additionally // assert the relevant channel closing post conditions. - ht.CloseChannel(alice, chanPoint, false) + ht.CloseChannel(alice, chanPoint) } return aliceChannel, bobChannel, closeChan @@ -533,16 +529,25 @@ func testExternalFundingChanPoint(ht *lntemp.HarnessTest) { // First, we'll try to close the channel as Carol, the initiator. This // should fail as a frozen channel only allows the responder to // initiate a channel close. - ht.CloseChannelAssertErr(carol, chanPoint2, false) + err := ht.CloseChannelAssertErr(carol, chanPoint2, false) + require.Contains(ht, err.Error(), "cannot co-op close frozen channel") // Before Dave closes the channel, he needs to check the invoice is // settled to avoid an error saying cannot close channel due to active // HTLCs. ht.AssertInvoiceSettled(dave, resp.PaymentAddr) + // TODO(yy): remove the sleep once the following bug is fixed. + // When the invoice is reported settled, the commitment dance is not + // yet finished, which can cause an error when closing the channel, + // saying there's active HTLCs. We need to investigate this issue and + // reverse the order to, first finish the commitment dance, then report + // the invoice as settled. + time.Sleep(2 * time.Second) + // Next we'll try but this time with Dave (the responder) as the // initiator. This time the channel should be closed as normal. - ht.CloseChannel(dave, chanPoint2, false) + ht.CloseChannel(dave, chanPoint2) // As a last step, we check if we still have the pending channel // hanging around because we never published the funding TX. diff --git a/lntest/itest/temp_lnd_test.go b/lntest/itest/temp_lnd_test.go index ee8e2ca71..48a617564 100644 --- a/lntest/itest/temp_lnd_test.go +++ b/lntest/itest/temp_lnd_test.go @@ -97,6 +97,7 @@ func TestLightningNetworkDaemonTemp(t *testing.T) { ht, cleanup := harnessTest.Subtest(t1) defer cleanup() + // TODO(yy): split log files. cleanTestCaseName := strings.ReplaceAll( testCase.Name, " ", "_", )