From cb4cd49dc8d3b0255afe9ff29af9c46c2dbb2c98 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Wed, 25 Mar 2020 14:59:08 +0100 Subject: [PATCH 01/13] routing: map insufficient local bandwidth error to no path With mpp it isn't possible anymore for findPath to determine that there isn't enough local bandwidth. The full payment amount isn't known at that point. In a follow-up, this payment outcome can be reintroduced on a higher level (payment lifecycle). --- channeldb/payments.go | 3 +++ routing/pathfind.go | 2 +- routing/pathfind_test.go | 31 ------------------------------- routing/payment_session.go | 10 ---------- 4 files changed, 4 insertions(+), 42 deletions(-) diff --git a/channeldb/payments.go b/channeldb/payments.go index 7c5f49a2a..16497dcdc 100644 --- a/channeldb/payments.go +++ b/channeldb/payments.go @@ -116,6 +116,9 @@ const ( // FailureReasonInsufficientBalance indicates that we didn't have enough // balance to complete the payment. + // + // This reason isn't assigned anymore, but may still exist for older + // payments. FailureReasonInsufficientBalance FailureReason = 4 // TODO(halseth): cancel state. diff --git a/routing/pathfind.go b/routing/pathfind.go index 1365958c6..3241fb121 100644 --- a/routing/pathfind.go +++ b/routing/pathfind.go @@ -478,7 +478,7 @@ func findPathInternal( return nil, err } if max < amt { - return nil, errInsufficientBalance + return nil, errNoPathFound } } diff --git a/routing/pathfind_test.go b/routing/pathfind_test.go index 686621198..deaa4ea53 100644 --- a/routing/pathfind_test.go +++ b/routing/pathfind_test.go @@ -2809,37 +2809,6 @@ func TestRouteToSelf(t *testing.T) { ctx.assertPath(path, []uint64{1, 3, 2}) } -// TestInsufficientBalance tests that a dedicated error is returned for -// insufficient local balance. -func TestInsufficientBalance(t *testing.T) { - t.Parallel() - - testChannels := []*testChannel{ - symmetricTestChannel("source", "target", 100000, &testChannelPolicy{ - Expiry: 144, - FeeBaseMsat: 500, - }, 1), - } - - ctx := newPathFindingTestContext(t, testChannels, "source") - defer ctx.cleanup() - - paymentAmt := lnwire.NewMSatFromSatoshis(100) - target := ctx.keyFromAlias("target") - - ctx.graphParams.bandwidthHints = map[uint64]lnwire.MilliSatoshi{ - 1: lnwire.NewMSatFromSatoshis(50), - } - - // Find the best path to self. We expect this to be source->a->source, - // because a charges the lowest forwarding fee. - _, err := ctx.findPath(target, paymentAmt) - if err != errInsufficientBalance { - t.Fatalf("expected insufficient balance error, but got: %v", - err) - } -} - type pathFindingTestContext struct { t *testing.T graphParams graphParams diff --git a/routing/payment_session.go b/routing/payment_session.go index 6a597ff01..b439420fb 100644 --- a/routing/payment_session.go +++ b/routing/payment_session.go @@ -26,10 +26,6 @@ const ( // not exist in the graph. errNoPathFound - // errInsufficientLocalBalance is returned when none of the local - // channels have enough balance for the payment. - errInsufficientBalance - // errEmptyPaySession is returned when the empty payment session is // queried for a route. errEmptyPaySession @@ -50,9 +46,6 @@ func (e noRouteError) Error() string { case errEmptyPaySession: return "empty payment session" - case errInsufficientBalance: - return "insufficient local balance" - default: return "unknown no-route error" } @@ -69,9 +62,6 @@ func (e noRouteError) FailureReason() channeldb.FailureReason { return channeldb.FailureReasonNoRoute - case errInsufficientBalance: - return channeldb.FailureReasonInsufficientBalance - default: return channeldb.FailureReasonError } From 47f9c1c3fdfcd82544162b020f44b6c02ba3f7df Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Tue, 17 Mar 2020 11:32:07 +0100 Subject: [PATCH 02/13] routing: use routingGraph interface in payment session Preparation for more test coverage of payment session. The function findPath now has the call signature of the former findPathInternal function. --- routing/integrated_routing_context_test.go | 7 +- routing/pathfind.go | 46 ++------ routing/pathfind_test.go | 126 ++++++++++----------- routing/payment_session.go | 29 +++-- routing/payment_session_source.go | 25 +++- routing/payment_session_test.go | 35 +++--- routing/router.go | 18 ++- routing/router_test.go | 14 +-- server.go | 1 - 9 files changed, 155 insertions(+), 146 deletions(-) diff --git a/routing/integrated_routing_context_test.go b/routing/integrated_routing_context_test.go index db5c6fe9f..b5fc955ee 100644 --- a/routing/integrated_routing_context_test.go +++ b/routing/integrated_routing_context_test.go @@ -111,8 +111,11 @@ func (c *integratedRoutingContext) testPayment(expectedNofAttempts int) { } // Find a route. - path, err := findPathInternal( - nil, bandwidthHints, c.graph, + path, err := findPath( + &graphParams{ + graph: c.graph, + bandwidthHints: bandwidthHints, + }, &restrictParams, &c.pathFindingCfg, c.source.pubkey, c.target.pubkey, diff --git a/routing/pathfind.go b/routing/pathfind.go index 3241fb121..5507444a0 100644 --- a/routing/pathfind.go +++ b/routing/pathfind.go @@ -253,7 +253,7 @@ func edgeWeight(lockedAmt lnwire.MilliSatoshi, fee lnwire.MilliSatoshi, // graphParams wraps the set of graph parameters passed to findPath. type graphParams struct { // graph is the ChannelGraph to be used during path finding. - graph *channeldb.ChannelGraph + graph routingGraph // additionalEdges is an optional set of edges that should be // considered during path finding, that is not already found in the @@ -381,34 +381,8 @@ func getMaxOutgoingAmt(node route.Vertex, outgoingChan *uint64, // path and accurately check the amount to forward at every node against the // available bandwidth. func findPath(g *graphParams, r *RestrictParams, cfg *PathFindingConfig, - source, target route.Vertex, amt lnwire.MilliSatoshi, finalHtlcExpiry int32) ( - []*channeldb.ChannelEdgePolicy, error) { - - routingTx, err := newDbRoutingTx(g.graph) - if err != nil { - return nil, err - } - defer func() { - err := routingTx.close() - if err != nil { - log.Errorf("Error closing db tx: %v", err) - } - }() - - return findPathInternal( - g.additionalEdges, g.bandwidthHints, routingTx, r, cfg, source, - target, amt, finalHtlcExpiry, - ) -} - -// findPathInternal is the internal implementation of findPath. -func findPathInternal( - additionalEdges map[route.Vertex][]*channeldb.ChannelEdgePolicy, - bandwidthHints map[uint64]lnwire.MilliSatoshi, - graph routingGraph, - r *RestrictParams, cfg *PathFindingConfig, - source, target route.Vertex, amt lnwire.MilliSatoshi, finalHtlcExpiry int32) ( - []*channeldb.ChannelEdgePolicy, error) { + source, target route.Vertex, amt lnwire.MilliSatoshi, + finalHtlcExpiry int32) ([]*channeldb.ChannelEdgePolicy, error) { // Pathfinding can be a significant portion of the total payment // latency, especially on low-powered devices. Log several metrics to @@ -427,7 +401,7 @@ func findPathInternal( features := r.DestFeatures if features == nil { var err error - features, err = graph.fetchNodeFeatures(target) + features, err = g.graph.fetchNodeFeatures(target) if err != nil { return nil, err } @@ -468,11 +442,11 @@ func findPathInternal( // If we are routing from ourselves, check that we have enough local // balance available. - self := graph.sourceNode() + self := g.graph.sourceNode() if source == self { max, err := getMaxOutgoingAmt( - self, r.OutgoingChannelID, bandwidthHints, graph, + self, r.OutgoingChannelID, g.bandwidthHints, g.graph, ) if err != nil { return nil, err @@ -491,7 +465,7 @@ func findPathInternal( distance := make(map[route.Vertex]*nodeWithDist, estimatedNodeCount) additionalEdgesWithSrc := make(map[route.Vertex][]*edgePolicyWithSource) - for vertex, outgoingEdgePolicies := range additionalEdges { + for vertex, outgoingEdgePolicies := range g.additionalEdges { // Build reverse lookup to find incoming edges. Needed because // search is taken place from target to source. for _, outgoingEdgePolicy := range outgoingEdgePolicies { @@ -739,7 +713,7 @@ func findPathInternal( } // Fetch node features fresh from the graph. - fromFeatures, err := graph.fetchNodeFeatures(node) + fromFeatures, err := g.graph.fetchNodeFeatures(node) if err != nil { return nil, err } @@ -775,7 +749,7 @@ func findPathInternal( // Create unified policies for all incoming connections. u := newUnifiedPolicies(self, pivot, r.OutgoingChannelID) - err := u.addGraphPolicies(graph) + err := u.addGraphPolicies(g.graph) if err != nil { return nil, err } @@ -806,7 +780,7 @@ func findPathInternal( } policy := unifiedPolicy.getPolicy( - amtToSend, bandwidthHints, + amtToSend, g.bandwidthHints, ) if policy == nil { diff --git a/routing/pathfind_test.go b/routing/pathfind_test.go index deaa4ea53..193e752d9 100644 --- a/routing/pathfind_test.go +++ b/routing/pathfind_test.go @@ -811,10 +811,8 @@ func testBasicGraphPathFindingCase(t *testing.T, graphInstance *testGraphInstanc paymentAmt := lnwire.NewMSatFromSatoshis(test.paymentAmt) target := graphInstance.aliasMap[test.target] - path, err := findPath( - &graphParams{ - graph: graphInstance.graph, - }, + path, err := dbFindPath( + graphInstance.graph, nil, nil, &RestrictParams{ FeeLimit: test.feeLimit, ProbabilitySource: noProbabilitySource, @@ -1005,11 +1003,8 @@ func TestPathFindingWithAdditionalEdges(t *testing.T) { find := func(r *RestrictParams) ( []*channeldb.ChannelEdgePolicy, error) { - return findPath( - &graphParams{ - graph: graph.graph, - additionalEdges: additionalEdges, - }, + return dbFindPath( + graph.graph, additionalEdges, nil, r, testPathFindingConfig, sourceNode.PubKeyBytes, doge.PubKeyBytes, paymentAmt, 0, @@ -1433,10 +1428,8 @@ func TestPathNotAvailable(t *testing.T) { var unknownNode route.Vertex copy(unknownNode[:], unknownNodeBytes) - _, err = findPath( - &graphParams{ - graph: graph.graph, - }, + _, err = dbFindPath( + graph.graph, nil, nil, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, unknownNode, 100, 0, ) @@ -1482,7 +1475,7 @@ func TestDestTLVGraphFallback(t *testing.T) { ctx := newPathFindingTestContext(t, testChannels, "roasbeef") defer ctx.cleanup() - sourceNode, err := ctx.graphParams.graph.SourceNode() + sourceNode, err := ctx.graph.SourceNode() if err != nil { t.Fatalf("unable to fetch source node: %v", err) @@ -1491,10 +1484,8 @@ func TestDestTLVGraphFallback(t *testing.T) { find := func(r *RestrictParams, target route.Vertex) ([]*channeldb.ChannelEdgePolicy, error) { - return findPath( - &graphParams{ - graph: ctx.graphParams.graph, - }, + return dbFindPath( + ctx.graph, nil, nil, r, testPathFindingConfig, sourceNode.PubKeyBytes, target, 100, 0, ) @@ -1765,10 +1756,8 @@ func TestPathInsufficientCapacity(t *testing.T) { target := graph.aliasMap["sophon"] payAmt := lnwire.NewMSatFromSatoshis(btcutil.SatoshiPerBitcoin) - _, err = findPath( - &graphParams{ - graph: graph.graph, - }, + _, err = dbFindPath( + graph.graph, nil, nil, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, target, payAmt, 0, ) @@ -1798,10 +1787,8 @@ func TestRouteFailMinHTLC(t *testing.T) { // attempt should fail. target := graph.aliasMap["songoku"] payAmt := lnwire.MilliSatoshi(10) - _, err = findPath( - &graphParams{ - graph: graph.graph, - }, + _, err = dbFindPath( + graph.graph, nil, nil, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, target, payAmt, 0, ) @@ -1897,10 +1884,8 @@ func TestRouteFailDisabledEdge(t *testing.T) { // succeed without issue, and return a single path via phamnuwen target := graph.aliasMap["sophon"] payAmt := lnwire.NewMSatFromSatoshis(105000) - _, err = findPath( - &graphParams{ - graph: graph.graph, - }, + _, err = dbFindPath( + graph.graph, nil, nil, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, target, payAmt, 0, ) @@ -1925,10 +1910,8 @@ func TestRouteFailDisabledEdge(t *testing.T) { t.Fatalf("unable to update edge: %v", err) } - _, err = findPath( - &graphParams{ - graph: graph.graph, - }, + _, err = dbFindPath( + graph.graph, nil, nil, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, target, payAmt, 0, ) @@ -1950,10 +1933,8 @@ func TestRouteFailDisabledEdge(t *testing.T) { // If we attempt to route through that edge, we should get a failure as // it is no longer eligible. - _, err = findPath( - &graphParams{ - graph: graph.graph, - }, + _, err = dbFindPath( + graph.graph, nil, nil, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, target, payAmt, 0, ) @@ -1984,10 +1965,8 @@ func TestPathSourceEdgesBandwidth(t *testing.T) { // cheapest path. target := graph.aliasMap["sophon"] payAmt := lnwire.NewMSatFromSatoshis(50000) - path, err := findPath( - &graphParams{ - graph: graph.graph, - }, + path, err := dbFindPath( + graph.graph, nil, nil, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, target, payAmt, 0, ) @@ -2007,11 +1986,8 @@ func TestPathSourceEdgesBandwidth(t *testing.T) { // Since both these edges has a bandwidth of zero, no path should be // found. - _, err = findPath( - &graphParams{ - graph: graph.graph, - bandwidthHints: bandwidths, - }, + _, err = dbFindPath( + graph.graph, nil, bandwidths, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, target, payAmt, 0, ) @@ -2025,11 +2001,8 @@ func TestPathSourceEdgesBandwidth(t *testing.T) { // Now, if we attempt to route again, we should find the path via // phamnuven, as the other source edge won't be considered. - path, err = findPath( - &graphParams{ - graph: graph.graph, - bandwidthHints: bandwidths, - }, + path, err = dbFindPath( + graph.graph, nil, bandwidths, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, target, payAmt, 0, ) @@ -2056,11 +2029,8 @@ func TestPathSourceEdgesBandwidth(t *testing.T) { // Since we ignore disable flags for local channels, a path should // still be found. - path, err = findPath( - &graphParams{ - graph: graph.graph, - bandwidthHints: bandwidths, - }, + path, err = dbFindPath( + graph.graph, nil, bandwidths, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, target, payAmt, 0, ) @@ -2811,7 +2781,7 @@ func TestRouteToSelf(t *testing.T) { type pathFindingTestContext struct { t *testing.T - graphParams graphParams + graph *channeldb.ChannelGraph restrictParams RestrictParams pathFindingConfig PathFindingConfig testGraphInstance *testGraphInstance @@ -2838,10 +2808,8 @@ func newPathFindingTestContext(t *testing.T, testChannels []*testChannel, testGraphInstance: testGraphInstance, source: route.Vertex(sourceNode.PubKeyBytes), pathFindingConfig: *testPathFindingConfig, - graphParams: graphParams{ - graph: testGraphInstance.graph, - }, - restrictParams: *noRestrictions, + graph: testGraphInstance.graph, + restrictParams: *noRestrictions, } return ctx @@ -2868,8 +2836,8 @@ func (c *pathFindingTestContext) findPath(target route.Vertex, amt lnwire.MilliSatoshi) ([]*channeldb.ChannelEdgePolicy, error) { - return findPath( - &c.graphParams, &c.restrictParams, &c.pathFindingConfig, + return dbFindPath( + c.graph, nil, nil, &c.restrictParams, &c.pathFindingConfig, c.source, target, amt, 0, ) } @@ -2887,3 +2855,33 @@ func (c *pathFindingTestContext) assertPath(path []*channeldb.ChannelEdgePolicy, } } } + +// dbFindPath calls findPath after getting a db transaction from the database +// graph. +func dbFindPath(graph *channeldb.ChannelGraph, + additionalEdges map[route.Vertex][]*channeldb.ChannelEdgePolicy, + bandwidthHints map[uint64]lnwire.MilliSatoshi, + r *RestrictParams, cfg *PathFindingConfig, + source, target route.Vertex, amt lnwire.MilliSatoshi, + finalHtlcExpiry int32) ([]*channeldb.ChannelEdgePolicy, error) { + + routingTx, err := newDbRoutingTx(graph) + if err != nil { + return nil, err + } + defer func() { + err := routingTx.close() + if err != nil { + log.Errorf("Error closing db tx: %v", err) + } + }() + + return findPath( + &graphParams{ + additionalEdges: additionalEdges, + bandwidthHints: bandwidthHints, + graph: routingTx, + }, + r, cfg, source, target, amt, finalHtlcExpiry, + ) +} diff --git a/routing/payment_session.go b/routing/payment_session.go index b439420fb..a28f6a6f1 100644 --- a/routing/payment_session.go +++ b/routing/payment_session.go @@ -98,13 +98,19 @@ type paymentSession struct { getBandwidthHints func() (map[uint64]lnwire.MilliSatoshi, error) - sessionSource *SessionSource - payment *LightningPayment empty bool pathFinder pathFinder + + getRoutingGraph func() (routingGraph, func(), error) + + // pathFindingConfig defines global parameters that control the + // trade-off in path finding between fees and probabiity. + pathFindingConfig PathFindingConfig + + missionControl MissionController } // RequestRoute returns a route which is likely to be capable for successfully @@ -138,10 +144,8 @@ func (p *paymentSession) RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi, // Taking into account this prune view, we'll attempt to locate a path // to our destination, respecting the recommendations from // MissionControl. - ss := p.sessionSource - restrictions := &RestrictParams{ - ProbabilitySource: ss.MissionControl.GetProbability, + ProbabilitySource: p.missionControl.GetProbability, FeeLimit: feeLimit, OutgoingChannelID: p.payment.OutgoingChannelID, LastHop: p.payment.LastHop, @@ -164,14 +168,22 @@ func (p *paymentSession) RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi, finalHtlcExpiry := int32(height) + int32(finalCltvDelta) + routingGraph, cleanup, err := p.getRoutingGraph() + if err != nil { + return nil, err + } + defer cleanup() + + sourceVertex := routingGraph.sourceNode() + path, err := p.pathFinder( &graphParams{ - graph: ss.Graph, additionalEdges: p.additionalEdges, bandwidthHints: bandwidthHints, + graph: routingGraph, }, - restrictions, &ss.PathFindingConfig, - ss.SelfNode.PubKeyBytes, p.payment.Target, + restrictions, &p.pathFindingConfig, + sourceVertex, p.payment.Target, maxAmt, finalHtlcExpiry, ) if err != nil { @@ -180,7 +192,6 @@ func (p *paymentSession) RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi, // With the next candidate path found, we'll attempt to turn this into // a route by applying the time-lock and fee requirements. - sourceVertex := route.Vertex(ss.SelfNode.PubKeyBytes) route, err := newRoute( sourceVertex, path, height, finalHopParams{ diff --git a/routing/payment_session_source.go b/routing/payment_session_source.go index 3d6cfedf2..f3b6607e1 100644 --- a/routing/payment_session_source.go +++ b/routing/payment_session_source.go @@ -26,9 +26,6 @@ type SessionSource struct { // the available bandwidth of the link should be returned. QueryBandwidth func(*channeldb.ChannelEdgeInfo) lnwire.MilliSatoshi - // SelfNode is our own node. - SelfNode *channeldb.LightningNode - // MissionControl is a shared memory of sorts that executions of payment // path finding use in order to remember which vertexes/edges were // pruned from prior attempts. During payment execution, errors sent by @@ -43,6 +40,21 @@ type SessionSource struct { PathFindingConfig PathFindingConfig } +// getRoutingGraph returns a routing graph and a clean-up function for +// pathfinding. +func (m *SessionSource) getRoutingGraph() (routingGraph, func(), error) { + routingTx, err := newDbRoutingTx(m.Graph) + if err != nil { + return nil, nil, err + } + return routingTx, func() { + err := routingTx.close() + if err != nil { + log.Errorf("Error closing db tx: %v", err) + } + }, nil +} + // NewPaymentSession creates a new payment session backed by the latest prune // view from Mission Control. An optional set of routing hints can be provided // in order to populate additional edges to explore when finding a path to the @@ -69,9 +81,11 @@ func (m *SessionSource) NewPaymentSession(p *LightningPayment) ( return &paymentSession{ additionalEdges: edges, getBandwidthHints: getBandwidthHints, - sessionSource: m, payment: p, pathFinder: findPath, + getRoutingGraph: m.getRoutingGraph, + pathFindingConfig: m.PathFindingConfig, + missionControl: m.MissionControl, }, nil } @@ -80,8 +94,7 @@ func (m *SessionSource) NewPaymentSession(p *LightningPayment) ( // missioncontrol for resumed payment we don't want to make more attempts for. func (m *SessionSource) NewPaymentSessionEmpty() PaymentSession { return &paymentSession{ - sessionSource: m, - empty: true, + empty: true, } } diff --git a/routing/payment_session_test.go b/routing/payment_session_test.go index a91658043..60d960b87 100644 --- a/routing/payment_session_test.go +++ b/routing/payment_session_test.go @@ -13,10 +13,11 @@ func TestRequestRoute(t *testing.T) { height = 10 ) - findPath := func(g *graphParams, r *RestrictParams, - cfg *PathFindingConfig, source, target route.Vertex, - amt lnwire.MilliSatoshi, finalHtlcExpiry int32) ( - []*channeldb.ChannelEdgePolicy, error) { + findPath := func( + g *graphParams, + r *RestrictParams, cfg *PathFindingConfig, + source, target route.Vertex, amt lnwire.MilliSatoshi, + finalHtlcExpiry int32) ([]*channeldb.ChannelEdgePolicy, error) { // We expect find path to receive a cltv limit excluding the // final cltv delta (including the block padding). @@ -37,13 +38,6 @@ func TestRequestRoute(t *testing.T) { return path, nil } - sessionSource := &SessionSource{ - SelfNode: &channeldb.LightningNode{}, - MissionControl: &MissionControl{ - cfg: &MissionControlConfig{}, - }, - } - cltvLimit := uint32(30) finalCltvDelta := uint16(8) @@ -60,9 +54,14 @@ func TestRequestRoute(t *testing.T) { return nil, nil }, - sessionSource: sessionSource, - payment: payment, - pathFinder: findPath, + payment: payment, + pathFinder: findPath, + missionControl: &MissionControl{ + cfg: &MissionControlConfig{}, + }, + getRoutingGraph: func() (routingGraph, func(), error) { + return &sessionGraph{}, func() {}, nil + }, } route, err := session.RequestRoute( @@ -79,3 +78,11 @@ func TestRequestRoute(t *testing.T) { route.TotalTimeLock) } } + +type sessionGraph struct { + routingGraph +} + +func (g *sessionGraph) sourceNode() route.Vertex { + return route.Vertex{} +} diff --git a/routing/router.go b/routing/router.go index cac3e69d4..c554ceee6 100644 --- a/routing/router.go +++ b/routing/router.go @@ -1426,13 +1426,25 @@ func (r *ChannelRouter) FindRoute(source, target route.Vertex, // execute our path finding algorithm. finalHtlcExpiry := currentHeight + int32(finalExpiry) + routingTx, err := newDbRoutingTx(r.cfg.Graph) + if err != nil { + return nil, err + } + defer func() { + err := routingTx.close() + if err != nil { + log.Errorf("Error closing db tx: %v", err) + } + }() + path, err := findPath( &graphParams{ - graph: r.cfg.Graph, - bandwidthHints: bandwidthHints, additionalEdges: routeHints, + bandwidthHints: bandwidthHints, + graph: routingTx, }, - restrictions, &r.cfg.PathFindingConfig, + restrictions, + &r.cfg.PathFindingConfig, source, target, amt, finalHtlcExpiry, ) if err != nil { diff --git a/routing/router_test.go b/routing/router_test.go index 7ac7527c5..d8b0950f0 100644 --- a/routing/router_test.go +++ b/routing/router_test.go @@ -79,11 +79,6 @@ func createTestCtxFromGraphInstance(startingHeight uint32, graphInstance *testGr chain := newMockChain(startingHeight) chainView := newMockChainView(chain) - selfNode, err := graphInstance.graph.SourceNode() - if err != nil { - return nil, nil, err - } - pathFindingConfig := PathFindingConfig{ MinProbability: 0.01, PaymentAttemptPenalty: 100, @@ -104,8 +99,7 @@ func createTestCtxFromGraphInstance(startingHeight uint32, graphInstance *testGr } sessionSource := &SessionSource{ - Graph: graphInstance.graph, - SelfNode: selfNode, + Graph: graphInstance.graph, QueryBandwidth: func(e *channeldb.ChannelEdgeInfo) lnwire.MilliSatoshi { return lnwire.NewMSatFromSatoshis(e.Capacity) }, @@ -2188,10 +2182,8 @@ func TestFindPathFeeWeighting(t *testing.T) { // We'll now attempt a path finding attempt using this set up. Due to // the edge weighting, we should select the direct path over the 2 hop // path even though the direct path has a higher potential time lock. - path, err := findPath( - &graphParams{ - graph: ctx.graph, - }, + path, err := dbFindPath( + ctx.graph, nil, nil, noRestrictions, testPathFindingConfig, sourceNode.PubKeyBytes, target, amt, 0, diff --git a/server.go b/server.go index a1dce6ec0..b56e43adc 100644 --- a/server.go +++ b/server.go @@ -731,7 +731,6 @@ func newServer(listenAddrs []net.Addr, chanDB *channeldb.DB, Graph: chanGraph, MissionControl: s.missionControl, QueryBandwidth: queryBandwidth, - SelfNode: selfNode, PathFindingConfig: pathFindingConfig, } From b13616a593c4210e5f7baac006369dc5068aac4f Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Tue, 17 Mar 2020 11:53:29 +0100 Subject: [PATCH 03/13] routing/test: use payment session in integrated routing test Cover the payment session in the integrated routing test as a preparation for executing the upcoming mpp split logic as part of the test. --- routing/integrated_routing_context_test.go | 50 ++++++++++++---------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/routing/integrated_routing_context_test.go b/routing/integrated_routing_context_test.go index b5fc955ee..eb66ac37a 100644 --- a/routing/integrated_routing_context_test.go +++ b/routing/integrated_routing_context_test.go @@ -95,10 +95,31 @@ func (c *integratedRoutingContext) testPayment(expectedNofAttempts int) { c.t.Fatal(err) } - // Instruct pathfinding to use mission control as a probabiltiy source. - restrictParams := RestrictParams{ - ProbabilitySource: mc.GetProbability, - FeeLimit: lnwire.MaxMilliSatoshi, + getBandwidthHints := func() (map[uint64]lnwire.MilliSatoshi, error) { + // Create bandwidth hints based on local channel balances. + bandwidthHints := map[uint64]lnwire.MilliSatoshi{} + for _, ch := range c.graph.nodes[c.source.pubkey].channels { + bandwidthHints[ch.id] = ch.balance + } + + return bandwidthHints, nil + } + + payment := LightningPayment{ + FinalCLTVDelta: uint16(c.finalExpiry), + FeeLimit: lnwire.MaxMilliSatoshi, + Target: c.target.pubkey, + } + + session := &paymentSession{ + getBandwidthHints: getBandwidthHints, + payment: &payment, + pathFinder: findPath, + getRoutingGraph: func() (routingGraph, func(), error) { + return c.graph, func() {}, nil + }, + pathFindingConfig: c.pathFindingCfg, + missionControl: mc, } // Now the payment control loop starts. It will keep trying routes until @@ -111,30 +132,13 @@ func (c *integratedRoutingContext) testPayment(expectedNofAttempts int) { } // Find a route. - path, err := findPath( - &graphParams{ - graph: c.graph, - bandwidthHints: bandwidthHints, - }, - &restrictParams, - &c.pathFindingCfg, - c.source.pubkey, c.target.pubkey, - c.amt, c.finalExpiry, + route, err := session.RequestRoute( + c.amt, lnwire.MaxMilliSatoshi, 0, 0, ) if err != nil { c.t.Fatal(err) } - finalHop := finalHopParams{ - amt: c.amt, - cltvDelta: uint16(c.finalExpiry), - } - - route, err := newRoute(c.source.pubkey, path, 0, finalHop) - if err != nil { - c.t.Fatal(err) - } - // Send out the htlc on the mock graph. pid := nextPid nextPid++ From 3c6e4612ffa77c031301c480e61d55fc4db0eaaa Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Tue, 17 Mar 2020 16:51:42 +0100 Subject: [PATCH 04/13] routing/test: expose full list of htlc attempts --- routing/integrated_routing_context_test.go | 29 ++++++++++++++++------ routing/integrated_routing_test.go | 10 ++++++-- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/routing/integrated_routing_context_test.go b/routing/integrated_routing_context_test.go index eb66ac37a..96c85d5f2 100644 --- a/routing/integrated_routing_context_test.go +++ b/routing/integrated_routing_context_test.go @@ -68,10 +68,19 @@ func newIntegratedRoutingContext(t *testing.T) *integratedRoutingContext { return &ctx } +// htlcAttempt records the route and outcome of an attempted htlc. +type htlcAttempt struct { + route *route.Route + success bool +} + // testPayment launches a test payment and asserts that it is completed after // the expected number of attempts. -func (c *integratedRoutingContext) testPayment(expectedNofAttempts int) { - var nextPid uint64 +func (c *integratedRoutingContext) testPayment() []htlcAttempt { + var ( + nextPid uint64 + attempts []htlcAttempt + ) // Create temporary database for mission control. file, err := ioutil.TempFile("", "*.db") @@ -147,8 +156,14 @@ func (c *integratedRoutingContext) testPayment(expectedNofAttempts int) { c.t.Fatal(err) } + success := htlcResult.failure == nil + attempts = append(attempts, htlcAttempt{ + route: route, + success: success, + }) + // Process the result. - if htlcResult.failure == nil { + if success { err := mc.ReportPaymentSuccess(pid, route) if err != nil { c.t.Fatal(err) @@ -177,11 +192,9 @@ func (c *integratedRoutingContext) testPayment(expectedNofAttempts int) { } } - c.t.Logf("Payment attempts: %v\n", nextPid) - if expectedNofAttempts != int(nextPid) { - c.t.Fatalf("expected %v attempts, but needed %v", - expectedNofAttempts, nextPid) - } + c.t.Logf("Payment attempts: %v\n", len(attempts)) + + return attempts } // getNodeIndex returns the zero-based index of the given node in the route. diff --git a/routing/integrated_routing_test.go b/routing/integrated_routing_test.go index 19df17a35..ccd221173 100644 --- a/routing/integrated_routing_test.go +++ b/routing/integrated_routing_test.go @@ -47,11 +47,17 @@ func TestProbabilityExtrapolation(t *testing.T) { // a specific number of attempts to safe-guard against accidental // modifications anywhere in the chain of components that is involved in // this test. - ctx.testPayment(5) + attempts := ctx.testPayment() + if len(attempts) != 5 { + t.Fatalf("expected 5 attempts, but needed %v", len(attempts)) + } // If we use a static value for the node probability (no extrapolation // of data from other channels), all ten bad channels will be tried // first before switching to the paid channel. ctx.mcCfg.AprioriWeight = 1 - ctx.testPayment(11) + attempts = ctx.testPayment() + if len(attempts) != 11 { + t.Fatalf("expected 11 attempts, but needed %v", len(attempts)) + } } From 0f8eb80965340773b893e5dc1211f3d452fcdf2b Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Tue, 17 Mar 2020 17:07:52 +0100 Subject: [PATCH 05/13] routing/test: use fixed identifiers in test graph --- routing/integrated_routing_context_test.go | 9 ++++- routing/integrated_routing_test.go | 19 ++++++---- routing/mock_graph_test.go | 44 +++++++++++++--------- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/routing/integrated_routing_context_test.go b/routing/integrated_routing_context_test.go index 96c85d5f2..d421c62fc 100644 --- a/routing/integrated_routing_context_test.go +++ b/routing/integrated_routing_context_test.go @@ -11,6 +11,11 @@ import ( "github.com/lightningnetwork/lnd/routing/route" ) +const ( + sourceNodeID = 1 + targetNodeID = 2 +) + // integratedRoutingContext defines the context in which integrated routing // tests run. type integratedRoutingContext struct { @@ -31,8 +36,8 @@ type integratedRoutingContext struct { // context with a source and a target node. func newIntegratedRoutingContext(t *testing.T) *integratedRoutingContext { // Instantiate a mock graph. - source := newMockNode() - target := newMockNode() + source := newMockNode(sourceNodeID) + target := newMockNode(targetNodeID) graph := newMockGraph(t) graph.addNode(source) diff --git a/routing/integrated_routing_test.go b/routing/integrated_routing_test.go index ccd221173..0362f208c 100644 --- a/routing/integrated_routing_test.go +++ b/routing/integrated_routing_test.go @@ -15,22 +15,25 @@ func TestProbabilityExtrapolation(t *testing.T) { // source -> intermediate1 (free routing) -> intermediate(1-10) (free routing) -> target g := ctx.graph - expensiveNode := newMockNode() + const expensiveNodeID = 3 + expensiveNode := newMockNode(expensiveNodeID) expensiveNode.baseFee = 10000 g.addNode(expensiveNode) - g.addChannel(ctx.source, expensiveNode, 100000) - g.addChannel(ctx.target, expensiveNode, 100000) + g.addChannel(100, sourceNodeID, expensiveNodeID, 100000) + g.addChannel(101, targetNodeID, expensiveNodeID, 100000) - intermediate1 := newMockNode() + const intermediate1NodeID = 4 + intermediate1 := newMockNode(intermediate1NodeID) g.addNode(intermediate1) - g.addChannel(ctx.source, intermediate1, 100000) + g.addChannel(102, sourceNodeID, intermediate1NodeID, 100000) for i := 0; i < 10; i++ { - imNode := newMockNode() + imNodeID := byte(10 + i) + imNode := newMockNode(imNodeID) g.addNode(imNode) - g.addChannel(imNode, ctx.target, 100000) - g.addChannel(imNode, intermediate1, 100000) + g.addChannel(uint64(200+i), imNodeID, targetNodeID, 100000) + g.addChannel(uint64(300+i), imNodeID, intermediate1NodeID, 100000) // The channels from intermediate1 all have insufficient balance. g.nodes[intermediate1.pubkey].channels[imNode.pubkey].balance = 0 diff --git a/routing/mock_graph_test.go b/routing/mock_graph_test.go index 075a416a1..3834d9e51 100644 --- a/routing/mock_graph_test.go +++ b/routing/mock_graph_test.go @@ -11,14 +11,9 @@ import ( "github.com/lightningnetwork/lnd/routing/route" ) -// nextTestPubkey is global variable that is used to deterministically generate -// test keys. -var nextTestPubkey byte - // createPubkey return a new test pubkey. -func createPubkey() route.Vertex { - pubkey := route.Vertex{nextTestPubkey} - nextTestPubkey++ +func createPubkey(id byte) route.Vertex { + pubkey := route.Vertex{id} return pubkey } @@ -38,8 +33,8 @@ type mockNode struct { } // newMockNode instantiates a new mock node with a newly generated pubkey. -func newMockNode() *mockNode { - pubkey := createPubkey() +func newMockNode(id byte) *mockNode { + pubkey := createPubkey(id) return &mockNode{ channels: make(map[route.Vertex]*mockChannel), pubkey: pubkey, @@ -106,10 +101,9 @@ func (m *mockNode) fwd(from *mockNode, route *hop) (htlcResult, error) { // mockGraph contains a set of nodes that together for a mocked graph. type mockGraph struct { - t *testing.T - nodes map[route.Vertex]*mockNode - nextChanID uint64 - source *mockNode + t *testing.T + nodes map[route.Vertex]*mockNode + source *mockNode } // newMockGraph instantiates a new mock graph. @@ -122,6 +116,11 @@ func newMockGraph(t *testing.T) *mockGraph { // addNode adds the given mock node to the network. func (m *mockGraph) addNode(node *mockNode) { + m.t.Helper() + + if _, exists := m.nodes[node.pubkey]; exists { + m.t.Fatal("node already exists") + } m.nodes[node.pubkey] = node } @@ -131,16 +130,25 @@ func (m *mockGraph) addNode(node *mockNode) { // Ignore linter error because addChannel isn't yet called with different // capacities. // nolint:unparam -func (m *mockGraph) addChannel(node1, node2 *mockNode, capacity btcutil.Amount) { - id := m.nextChanID - m.nextChanID++ +func (m *mockGraph) addChannel(id uint64, node1id, node2id byte, + capacity btcutil.Amount) { - m.nodes[node1.pubkey].channels[node2.pubkey] = &mockChannel{ + node1pubkey := createPubkey(node1id) + node2pubkey := createPubkey(node2id) + + if _, exists := m.nodes[node1pubkey].channels[node2pubkey]; exists { + m.t.Fatal("channel already exists") + } + if _, exists := m.nodes[node2pubkey].channels[node1pubkey]; exists { + m.t.Fatal("channel already exists") + } + + m.nodes[node1pubkey].channels[node2pubkey] = &mockChannel{ capacity: capacity, id: id, balance: lnwire.NewMSatFromSatoshis(capacity / 2), } - m.nodes[node2.pubkey].channels[node1.pubkey] = &mockChannel{ + m.nodes[node2pubkey].channels[node1pubkey] = &mockChannel{ capacity: capacity, id: id, balance: lnwire.NewMSatFromSatoshis(capacity / 2), From e5c7e9a38c79354178ae0366c0582c9a80fb83fd Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Wed, 18 Mar 2020 10:51:07 +0100 Subject: [PATCH 06/13] routing/test: return pathfinding error --- routing/integrated_routing_context_test.go | 6 +++--- routing/integrated_routing_test.go | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/routing/integrated_routing_context_test.go b/routing/integrated_routing_context_test.go index d421c62fc..418b380ca 100644 --- a/routing/integrated_routing_context_test.go +++ b/routing/integrated_routing_context_test.go @@ -81,7 +81,7 @@ type htlcAttempt struct { // testPayment launches a test payment and asserts that it is completed after // the expected number of attempts. -func (c *integratedRoutingContext) testPayment() []htlcAttempt { +func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) { var ( nextPid uint64 attempts []htlcAttempt @@ -150,7 +150,7 @@ func (c *integratedRoutingContext) testPayment() []htlcAttempt { c.amt, lnwire.MaxMilliSatoshi, 0, 0, ) if err != nil { - c.t.Fatal(err) + return nil, err } // Send out the htlc on the mock graph. @@ -199,7 +199,7 @@ func (c *integratedRoutingContext) testPayment() []htlcAttempt { c.t.Logf("Payment attempts: %v\n", len(attempts)) - return attempts + return attempts, nil } // getNodeIndex returns the zero-based index of the given node in the route. diff --git a/routing/integrated_routing_test.go b/routing/integrated_routing_test.go index 0362f208c..1e77ce810 100644 --- a/routing/integrated_routing_test.go +++ b/routing/integrated_routing_test.go @@ -50,7 +50,10 @@ func TestProbabilityExtrapolation(t *testing.T) { // a specific number of attempts to safe-guard against accidental // modifications anywhere in the chain of components that is involved in // this test. - attempts := ctx.testPayment() + attempts, err := ctx.testPayment() + if err != nil { + t.Fatalf("payment failed: %v", err) + } if len(attempts) != 5 { t.Fatalf("expected 5 attempts, but needed %v", len(attempts)) } @@ -59,7 +62,10 @@ func TestProbabilityExtrapolation(t *testing.T) { // of data from other channels), all ten bad channels will be tried // first before switching to the paid channel. ctx.mcCfg.AprioriWeight = 1 - attempts = ctx.testPayment() + attempts, err = ctx.testPayment() + if err != nil { + t.Fatalf("payment failed: %v", err) + } if len(attempts) != 11 { t.Fatalf("expected 11 attempts, but needed %v", len(attempts)) } From 46f5fc740036a1a68ace56e46906a51c0e9892d8 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Wed, 25 Mar 2020 14:06:48 +0100 Subject: [PATCH 07/13] routing: distinguish between receiver amount and total amount in newRoute --- routing/pathfind.go | 4 +++- routing/pathfind_test.go | 1 + routing/payment_session.go | 1 + routing/router.go | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/routing/pathfind.go b/routing/pathfind.go index 5507444a0..b6879cc6d 100644 --- a/routing/pathfind.go +++ b/routing/pathfind.go @@ -74,6 +74,7 @@ type edgePolicyWithSource struct { // custom records and payment address. type finalHopParams struct { amt lnwire.MilliSatoshi + totalAmt lnwire.MilliSatoshi cltvDelta uint16 records record.CustomSet paymentAddr *[32]byte @@ -177,7 +178,8 @@ func newRoute(sourceVertex route.Vertex, // Otherwise attach the mpp record if it exists. if finalHop.paymentAddr != nil { mpp = record.NewMPP( - finalHop.amt, *finalHop.paymentAddr, + finalHop.totalAmt, + *finalHop.paymentAddr, ) } } else { diff --git a/routing/pathfind_test.go b/routing/pathfind_test.go index 193e752d9..ab54534bd 100644 --- a/routing/pathfind_test.go +++ b/routing/pathfind_test.go @@ -1327,6 +1327,7 @@ func TestNewRoute(t *testing.T) { sourceVertex, testCase.hops, startingHeight, finalHopParams{ amt: testCase.paymentAmount, + totalAmt: testCase.paymentAmount, cltvDelta: finalHopCLTV, records: nil, paymentAddr: testCase.paymentAddr, diff --git a/routing/payment_session.go b/routing/payment_session.go index a28f6a6f1..9f9c347c9 100644 --- a/routing/payment_session.go +++ b/routing/payment_session.go @@ -196,6 +196,7 @@ func (p *paymentSession) RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi, sourceVertex, path, height, finalHopParams{ amt: maxAmt, + totalAmt: maxAmt, cltvDelta: finalCltvDelta, records: p.payment.DestCustomRecords, paymentAddr: p.payment.PaymentAddr, diff --git a/routing/router.go b/routing/router.go index c554ceee6..7d80305b9 100644 --- a/routing/router.go +++ b/routing/router.go @@ -1456,6 +1456,7 @@ func (r *ChannelRouter) FindRoute(source, target route.Vertex, source, path, uint32(currentHeight), finalHopParams{ amt: amt, + totalAmt: amt, cltvDelta: finalExpiry, records: destCustomRecords, }, @@ -2513,6 +2514,7 @@ func (r *ChannelRouter) BuildRoute(amt *lnwire.MilliSatoshi, source, pathEdges, uint32(height), finalHopParams{ amt: receiverAmt, + totalAmt: receiverAmt, cltvDelta: uint16(finalCltvDelta), records: nil, }, From e9bd691e6a17a0c4c9ff3fd99f60d855bd3cad9b Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Tue, 28 Jan 2020 16:07:34 +0100 Subject: [PATCH 08/13] routerrpc+routing: adapt payment session for multi shard send Modifies the payment session to launch additional pathfinding attempts for lower amounts. If a single shot payment isn't possible, the goal is to try to complete the payment using multiple htlcs. In previous commits, the payment lifecycle has been prepared to deal with partial-amount routes returned from the payment session. It will query for additional shards if needed. Additionally a new rpc payment parameter is added that controls the maximum number of shards that will be used for the payment. --- lnrpc/routerrpc/router.pb.go | 276 +++++++++++---------- lnrpc/routerrpc/router.proto | 6 + lnrpc/routerrpc/router_backend.go | 8 + routing/integrated_routing_context_test.go | 53 +++- routing/integrated_routing_test.go | 215 +++++++++++++++- routing/pathfind_test.go | 6 + routing/payment_session.go | 144 +++++++---- routing/payment_session_source.go | 1 + routing/router.go | 4 + rpcserver.go | 4 + 10 files changed, 520 insertions(+), 197 deletions(-) diff --git a/lnrpc/routerrpc/router.pb.go b/lnrpc/routerrpc/router.pb.go index 60f70e5e9..018c3d8c3 100644 --- a/lnrpc/routerrpc/router.pb.go +++ b/lnrpc/routerrpc/router.pb.go @@ -221,10 +221,14 @@ type SendPaymentRequest struct { //optional or remote may be set, but not both. If this field is nil or empty, //the router will try to load destination features from the graph as a //fallback. - DestFeatures []lnrpc.FeatureBit `protobuf:"varint,16,rep,packed,name=dest_features,json=destFeatures,proto3,enum=lnrpc.FeatureBit" json:"dest_features,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + DestFeatures []lnrpc.FeatureBit `protobuf:"varint,16,rep,packed,name=dest_features,json=destFeatures,proto3,enum=lnrpc.FeatureBit" json:"dest_features,omitempty"` + //* + //The maximum number of partial payments that may be use to complete the full + //amount. + MaxHtlcs uint32 `protobuf:"varint,17,opt,name=max_htlcs,json=maxHtlcs,proto3" json:"max_htlcs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *SendPaymentRequest) Reset() { *m = SendPaymentRequest{} } @@ -364,6 +368,13 @@ func (m *SendPaymentRequest) GetDestFeatures() []lnrpc.FeatureBit { return nil } +func (m *SendPaymentRequest) GetMaxHtlcs() uint32 { + if m != nil { + return m.MaxHtlcs + } + return 0 +} + type TrackPaymentRequest struct { /// The hash of the payment to look up. PaymentHash []byte `protobuf:"bytes,1,opt,name=payment_hash,json=paymentHash,proto3" json:"payment_hash,omitempty"` @@ -1595,134 +1606,135 @@ func init() { func init() { proto.RegisterFile("routerrpc/router.proto", fileDescriptor_7a0613f69d37b0a5) } var fileDescriptor_7a0613f69d37b0a5 = []byte{ - // 2017 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x58, 0x5d, 0x77, 0xdb, 0x48, - 0x19, 0x5e, 0xc5, 0x72, 0x62, 0xbf, 0xfe, 0x52, 0xc6, 0xdd, 0xd4, 0x38, 0xed, 0xe2, 0x15, 0x6c, - 0xeb, 0x53, 0x4a, 0xd2, 0x0d, 0x1c, 0xe8, 0x01, 0xb6, 0xe0, 0x58, 0xca, 0x46, 0x8d, 0x23, 0x79, - 0xc7, 0x4e, 0xdb, 0xa5, 0x17, 0x73, 0x14, 0x7b, 0x1c, 0x8b, 0xc8, 0x92, 0x91, 0xc6, 0xed, 0xc9, - 0x25, 0xb7, 0xfc, 0x11, 0xfe, 0x04, 0xff, 0x85, 0x5b, 0xee, 0xb8, 0xe3, 0x70, 0xc9, 0x99, 0x91, - 0xc6, 0x96, 0x13, 0xa7, 0xbb, 0x37, 0x89, 0xf4, 0xbc, 0xcf, 0xbc, 0x1f, 0xf3, 0x3e, 0xf3, 0x21, - 0xc3, 0x5e, 0x14, 0x2e, 0x18, 0x8d, 0xa2, 0xf9, 0xe8, 0x30, 0x79, 0x3a, 0x98, 0x47, 0x21, 0x0b, - 0x51, 0x71, 0x89, 0x37, 0x8b, 0xd1, 0x7c, 0x94, 0xa0, 0xfa, 0xff, 0xf2, 0x80, 0x06, 0x34, 0x18, - 0xf7, 0xdd, 0x9b, 0x19, 0x0d, 0x18, 0xa6, 0x7f, 0x5d, 0xd0, 0x98, 0x21, 0x04, 0xea, 0x98, 0xc6, - 0xac, 0xa1, 0xb4, 0x94, 0x76, 0x19, 0x8b, 0x67, 0xa4, 0x41, 0xce, 0x9d, 0xb1, 0xc6, 0x56, 0x4b, - 0x69, 0xe7, 0x30, 0x7f, 0x44, 0x3f, 0x81, 0x82, 0x3b, 0x63, 0x64, 0x16, 0xbb, 0xac, 0x51, 0x16, - 0xf0, 0x8e, 0x3b, 0x63, 0xe7, 0xb1, 0xcb, 0xd0, 0x97, 0x50, 0x9e, 0x27, 0x2e, 0xc9, 0xd4, 0x8d, - 0xa7, 0x8d, 0x9c, 0x70, 0x54, 0x4a, 0xb1, 0x53, 0x37, 0x9e, 0xa2, 0x36, 0x68, 0x13, 0x2f, 0x70, - 0x7d, 0x32, 0xf2, 0xd9, 0x07, 0x32, 0xa6, 0x3e, 0x73, 0x1b, 0x6a, 0x4b, 0x69, 0xe7, 0x71, 0x55, - 0xe0, 0x5d, 0x9f, 0x7d, 0x30, 0x38, 0x8a, 0x9e, 0x42, 0x4d, 0x3a, 0x8b, 0x92, 0x04, 0x1b, 0xf9, - 0x96, 0xd2, 0x2e, 0xe2, 0xea, 0x7c, 0x3d, 0xed, 0xa7, 0x50, 0x63, 0xde, 0x8c, 0x86, 0x0b, 0x46, - 0x62, 0x3a, 0x0a, 0x83, 0x71, 0xdc, 0xd8, 0x4e, 0x3c, 0xa6, 0xf0, 0x20, 0x41, 0x91, 0x0e, 0x95, - 0x09, 0xa5, 0xc4, 0xf7, 0x66, 0x1e, 0x23, 0x3c, 0xfd, 0x1d, 0x91, 0x7e, 0x69, 0x42, 0x69, 0x8f, - 0x63, 0x03, 0x97, 0xa1, 0x9f, 0x43, 0x75, 0xc5, 0x11, 0x35, 0x56, 0x04, 0xa9, 0x2c, 0x49, 0xa2, - 0xd0, 0xe7, 0xa0, 0x85, 0x0b, 0x76, 0x15, 0x7a, 0xc1, 0x15, 0x19, 0x4d, 0xdd, 0x80, 0x78, 0xe3, - 0x46, 0xa1, 0xa5, 0xb4, 0xd5, 0xe3, 0xad, 0x17, 0x0a, 0xae, 0x4a, 0x5b, 0x77, 0xea, 0x06, 0xd6, - 0x18, 0x3d, 0x81, 0x9a, 0xef, 0xc6, 0x8c, 0x4c, 0xc3, 0x39, 0x99, 0x2f, 0x2e, 0xaf, 0xe9, 0x4d, - 0xa3, 0x2a, 0x66, 0xa6, 0xc2, 0xe1, 0xd3, 0x70, 0xde, 0x17, 0x20, 0x7a, 0x0c, 0x20, 0x66, 0x45, - 0x04, 0x6f, 0x14, 0x45, 0x0d, 0x45, 0x8e, 0x88, 0xc0, 0xe8, 0x6b, 0x28, 0x89, 0x6e, 0x92, 0xa9, - 0x17, 0xb0, 0xb8, 0x01, 0xad, 0x5c, 0xbb, 0x74, 0xa4, 0x1d, 0xf8, 0x01, 0x6f, 0x2c, 0xe6, 0x96, - 0x53, 0x2f, 0x60, 0x18, 0x22, 0xf9, 0x18, 0xa3, 0x31, 0xd4, 0x79, 0x17, 0xc9, 0x68, 0x11, 0xb3, - 0x70, 0x46, 0x22, 0x3a, 0x0a, 0xa3, 0x71, 0xdc, 0x28, 0x89, 0xa1, 0xbf, 0x3e, 0x58, 0x8a, 0xe3, - 0xe0, 0xae, 0x1a, 0x0e, 0x0c, 0x1a, 0xb3, 0xae, 0x18, 0x87, 0x93, 0x61, 0x66, 0xc0, 0xa2, 0x1b, - 0xbc, 0x3b, 0xbe, 0x8d, 0xa3, 0xe7, 0x80, 0x5c, 0xdf, 0x0f, 0x3f, 0x92, 0x98, 0xfa, 0x13, 0x92, - 0x76, 0xa7, 0x51, 0x6b, 0x29, 0xed, 0x02, 0xd6, 0x84, 0x65, 0x40, 0xfd, 0x49, 0xea, 0x1e, 0xfd, - 0x06, 0x2a, 0x22, 0xa7, 0x09, 0x75, 0xd9, 0x22, 0xa2, 0x71, 0x43, 0x6b, 0xe5, 0xda, 0xd5, 0xa3, - 0xdd, 0xb4, 0x90, 0x93, 0x04, 0x3e, 0xf6, 0x18, 0x2e, 0x73, 0x5e, 0xfa, 0x1e, 0x37, 0x0d, 0xd8, - 0xdb, 0x9c, 0x12, 0xd7, 0x28, 0x9f, 0x53, 0x2e, 0x5b, 0x15, 0xf3, 0x47, 0xf4, 0x00, 0xf2, 0x1f, - 0x5c, 0x7f, 0x41, 0x85, 0x6e, 0xcb, 0x38, 0x79, 0xf9, 0xdd, 0xd6, 0x4b, 0x45, 0x7f, 0x09, 0xf5, - 0x61, 0xe4, 0x8e, 0xae, 0x6f, 0x49, 0xff, 0xb6, 0x72, 0x95, 0x3b, 0xca, 0xd5, 0x5f, 0x41, 0x4d, - 0x4c, 0xf2, 0x09, 0xa5, 0x9f, 0x5a, 0x30, 0x0f, 0x81, 0x2f, 0x07, 0x21, 0xaf, 0x64, 0xd1, 0x6c, - 0xbb, 0x33, 0xae, 0x2c, 0x7d, 0x0c, 0xda, 0x6a, 0x7c, 0x3c, 0x0f, 0x83, 0x98, 0xf2, 0xd5, 0xc0, - 0x7b, 0xc0, 0x65, 0xc4, 0x55, 0x27, 0xf4, 0xa6, 0x88, 0x51, 0xd5, 0x14, 0x3f, 0xa1, 0x54, 0x28, - 0xee, 0x49, 0x22, 0x72, 0xe2, 0x87, 0xa3, 0x6b, 0xbe, 0x6c, 0xdc, 0x9b, 0xd4, 0x7d, 0x85, 0xc3, - 0xbd, 0x70, 0x74, 0x6d, 0x70, 0x50, 0x7f, 0x9f, 0xac, 0xec, 0x61, 0x28, 0x62, 0xfd, 0xf8, 0xf2, - 0x90, 0x0e, 0x79, 0x21, 0x07, 0xe1, 0xb6, 0x74, 0x54, 0xce, 0xea, 0x0a, 0x27, 0x26, 0xfd, 0x3d, - 0xd4, 0xd7, 0x9c, 0xa7, 0x55, 0x34, 0xa1, 0x30, 0x8f, 0xa8, 0x37, 0x73, 0xaf, 0x68, 0xea, 0x79, - 0xf9, 0x8e, 0xda, 0xb0, 0x33, 0x71, 0x3d, 0x7f, 0x11, 0x49, 0xc7, 0x55, 0xd9, 0xe7, 0x04, 0xc5, - 0xd2, 0xac, 0x3f, 0x82, 0x26, 0xa6, 0x31, 0x65, 0xe7, 0x5e, 0x1c, 0x7b, 0x61, 0xd0, 0x0d, 0x03, - 0x16, 0x85, 0x7e, 0x5a, 0x81, 0xfe, 0x18, 0xf6, 0x37, 0x5a, 0x93, 0x14, 0xf8, 0xe0, 0xef, 0x16, - 0x34, 0xba, 0xd9, 0x3c, 0xf8, 0x3b, 0xd8, 0xdf, 0x68, 0x4d, 0xf3, 0x7f, 0x0e, 0xf9, 0xb9, 0xeb, - 0x45, 0x71, 0x63, 0x4b, 0xac, 0x8b, 0xbd, 0xcc, 0xba, 0xe8, 0xbb, 0x5e, 0x74, 0xea, 0xc5, 0x2c, - 0x8c, 0x6e, 0x70, 0x42, 0x7a, 0xad, 0x16, 0x14, 0x6d, 0x4b, 0xff, 0xbb, 0x02, 0xa5, 0x8c, 0x11, - 0xed, 0x43, 0x31, 0x08, 0xc7, 0x94, 0x4c, 0xa2, 0x70, 0x26, 0x27, 0x81, 0x03, 0x27, 0x51, 0x38, - 0xe3, 0x9a, 0x10, 0x46, 0x16, 0xa6, 0x82, 0xdc, 0xe6, 0xaf, 0xc3, 0x10, 0xfd, 0x12, 0x76, 0xa6, - 0x89, 0x03, 0xb1, 0x17, 0x95, 0x8e, 0xea, 0xb7, 0x62, 0x1b, 0x2e, 0x73, 0xb1, 0xe4, 0xbc, 0x56, - 0x0b, 0x39, 0x4d, 0x7d, 0xad, 0x16, 0x54, 0x2d, 0xff, 0x5a, 0x2d, 0xe4, 0xb5, 0xed, 0xd7, 0x6a, - 0x61, 0x5b, 0xdb, 0xd1, 0xff, 0xad, 0x40, 0x41, 0xb2, 0x79, 0x26, 0x7c, 0x4a, 0x09, 0xd7, 0x45, - 0x2a, 0xa6, 0x02, 0x07, 0x86, 0xde, 0x8c, 0xa2, 0x16, 0x94, 0x85, 0x71, 0x5d, 0xa2, 0xc0, 0xb1, - 0x8e, 0x90, 0xa9, 0xd8, 0x24, 0x25, 0x43, 0xe8, 0x51, 0x4d, 0x37, 0xc9, 0x84, 0x22, 0xf7, 0xf9, - 0x78, 0x31, 0x1a, 0xd1, 0x38, 0x4e, 0xa2, 0xe4, 0x13, 0x4a, 0x8a, 0x89, 0x40, 0x4f, 0xa0, 0x26, - 0x29, 0x32, 0xd6, 0x76, 0xa2, 0xd7, 0x14, 0x4e, 0xc3, 0xb5, 0x41, 0xcb, 0xf2, 0x66, 0xab, 0x6d, - 0xb9, 0xba, 0x22, 0xf2, 0xa0, 0x49, 0xf1, 0xfa, 0x5f, 0xe0, 0xa1, 0x68, 0x65, 0x3f, 0x0a, 0x2f, - 0xdd, 0x4b, 0xcf, 0xf7, 0xd8, 0x8d, 0x14, 0x39, 0x2f, 0x3c, 0x0a, 0x67, 0x84, 0xcf, 0xad, 0x6c, - 0x01, 0x07, 0xec, 0x70, 0x4c, 0x79, 0x0b, 0x58, 0x98, 0x98, 0xd2, 0x16, 0xb0, 0x50, 0x18, 0xb2, - 0xc7, 0x59, 0x6e, 0xed, 0x38, 0xd3, 0xaf, 0xa1, 0x71, 0x37, 0x56, 0xaa, 0x99, 0x16, 0x94, 0xe6, - 0x2b, 0x58, 0x84, 0x53, 0x70, 0x16, 0xca, 0xf6, 0x76, 0xeb, 0x87, 0x7b, 0xab, 0xff, 0x43, 0x81, - 0xdd, 0xe3, 0x85, 0xe7, 0x8f, 0xd7, 0x16, 0x6e, 0x36, 0x3b, 0x65, 0xfd, 0xb0, 0xdd, 0x74, 0x92, - 0x6e, 0x6d, 0x3c, 0x49, 0x37, 0x9d, 0x56, 0xb9, 0x7b, 0x4f, 0xab, 0x9f, 0x42, 0x69, 0x75, 0x50, - 0xc5, 0x0d, 0xb5, 0x95, 0x6b, 0x97, 0x31, 0x4c, 0xe5, 0x29, 0x15, 0xeb, 0x2f, 0x01, 0x65, 0x13, - 0x4d, 0x27, 0x64, 0xb9, 0x7f, 0x28, 0xf7, 0xef, 0x1f, 0x8f, 0xa0, 0x39, 0x58, 0x5c, 0xc6, 0xa3, - 0xc8, 0xbb, 0xa4, 0xa7, 0xcc, 0x1f, 0x99, 0x1f, 0x68, 0xc0, 0x62, 0xb9, 0x4a, 0xff, 0xab, 0x42, - 0x71, 0x89, 0xa2, 0x03, 0xa8, 0x7b, 0xc1, 0x28, 0x9c, 0xc9, 0xa4, 0x03, 0xea, 0xf3, 0xbc, 0x93, - 0x4d, 0x7e, 0x57, 0x9a, 0xba, 0x89, 0xc5, 0x1a, 0x73, 0xfe, 0x5a, 0x91, 0x29, 0x7f, 0x2b, 0xe1, - 0x67, 0x6b, 0x4c, 0xf8, 0x6d, 0xd0, 0x96, 0xfe, 0xa7, 0xcc, 0x1f, 0x2d, 0x27, 0x05, 0x57, 0x25, - 0xce, 0x93, 0x49, 0x98, 0x4b, 0xcf, 0x92, 0xa9, 0x26, 0x4c, 0x89, 0xa7, 0xcc, 0x2f, 0xa1, 0xcc, - 0xd7, 0x43, 0xcc, 0xdc, 0xd9, 0x9c, 0x04, 0xb1, 0x58, 0x17, 0x2a, 0x2e, 0x2d, 0x31, 0x3b, 0x46, - 0xdf, 0x00, 0x50, 0x5e, 0x1f, 0x61, 0x37, 0x73, 0x2a, 0x96, 0x44, 0xf5, 0xe8, 0x8b, 0x8c, 0x30, - 0x96, 0x13, 0x70, 0x20, 0xfe, 0x0e, 0x6f, 0xe6, 0x14, 0x17, 0xa9, 0x7c, 0x44, 0xaf, 0xa0, 0x32, - 0x09, 0xa3, 0x8f, 0x6e, 0x34, 0x26, 0x02, 0x4c, 0xb7, 0x8d, 0x87, 0x19, 0x0f, 0x27, 0x89, 0x5d, - 0x0c, 0x3f, 0xfd, 0x0c, 0x97, 0x27, 0x99, 0x77, 0x74, 0x06, 0x48, 0x8e, 0x17, 0xab, 0x3c, 0x71, - 0x52, 0x10, 0x4e, 0xf6, 0xef, 0x3a, 0xe1, 0x9b, 0xb4, 0x74, 0xa4, 0x4d, 0x6e, 0x61, 0xe8, 0xf7, - 0x50, 0x8e, 0x29, 0x63, 0x3e, 0x4d, 0xdd, 0x14, 0x85, 0x9b, 0xbd, 0xb5, 0x6b, 0x05, 0x37, 0x4b, - 0x0f, 0xa5, 0x78, 0xf5, 0x8a, 0x8e, 0xa1, 0xe6, 0x7b, 0xc1, 0x75, 0x36, 0x0d, 0x10, 0xe3, 0x1b, - 0x99, 0xf1, 0x3d, 0x2f, 0xb8, 0xce, 0xe6, 0x50, 0xf1, 0xb3, 0x80, 0xfe, 0x07, 0x28, 0x2e, 0x67, - 0x09, 0x95, 0x60, 0xe7, 0xc2, 0x3e, 0xb3, 0x9d, 0xb7, 0xb6, 0xf6, 0x19, 0x2a, 0x80, 0x3a, 0x30, - 0x6d, 0x43, 0x53, 0x38, 0x8c, 0xcd, 0xae, 0x69, 0xbd, 0x31, 0xb5, 0x2d, 0xfe, 0x72, 0xe2, 0xe0, - 0xb7, 0x1d, 0x6c, 0x68, 0xb9, 0xe3, 0x1d, 0xc8, 0x8b, 0xb8, 0xfa, 0x3f, 0x15, 0x28, 0x88, 0x0e, - 0x06, 0x93, 0x10, 0xfd, 0x02, 0x96, 0xe2, 0x12, 0x9b, 0x1b, 0x3f, 0x70, 0x85, 0xea, 0x2a, 0x78, - 0x29, 0x98, 0x61, 0x8a, 0x73, 0xf2, 0x52, 0x1a, 0x4b, 0xf2, 0x56, 0x42, 0x96, 0x86, 0x25, 0xf9, - 0x59, 0xc6, 0xf3, 0xda, 0x96, 0xa3, 0xe2, 0x9a, 0x34, 0xc8, 0x1d, 0xf6, 0x59, 0xc6, 0xf1, 0xda, - 0x4e, 0xac, 0xe2, 0x9a, 0x34, 0xa4, 0x5c, 0xfd, 0xb7, 0x50, 0xce, 0xf6, 0x1c, 0x3d, 0x05, 0xd5, - 0x0b, 0x26, 0x61, 0xba, 0x10, 0xeb, 0xb7, 0xc4, 0xc5, 0x8b, 0xc4, 0x82, 0xa0, 0x23, 0xd0, 0x6e, - 0xf7, 0x59, 0xaf, 0x40, 0x29, 0xd3, 0x34, 0xfd, 0x5f, 0x0a, 0x54, 0xd6, 0x9a, 0xf0, 0xa3, 0xbd, - 0xa3, 0x6f, 0xa0, 0xfc, 0xd1, 0x8b, 0x28, 0xc9, 0x1e, 0xff, 0xd5, 0xa3, 0xe6, 0xfa, 0xf1, 0x2f, - 0xff, 0x77, 0xc3, 0x31, 0xc5, 0x25, 0xce, 0x4f, 0x01, 0xf4, 0x47, 0xa8, 0xa6, 0x23, 0xc9, 0x98, - 0x32, 0xd7, 0xf3, 0xc5, 0x54, 0x55, 0xd7, 0xe4, 0x91, 0x72, 0x0d, 0x61, 0xc7, 0x95, 0x49, 0xf6, - 0x15, 0x7d, 0xb5, 0x72, 0x10, 0xb3, 0xc8, 0x0b, 0xae, 0xc4, 0xfc, 0x15, 0x97, 0xb4, 0x81, 0x00, - 0x9f, 0xfd, 0x4d, 0x85, 0xca, 0x9a, 0x9f, 0x75, 0x21, 0x55, 0xa0, 0x68, 0x3b, 0xc4, 0x30, 0x87, - 0x1d, 0xab, 0xa7, 0x29, 0x48, 0x83, 0xb2, 0x63, 0x5b, 0x8e, 0x4d, 0x0c, 0xb3, 0xeb, 0x18, 0x5c, - 0x52, 0x9f, 0xc3, 0x6e, 0xcf, 0xb2, 0xcf, 0x88, 0xed, 0x0c, 0x89, 0xd9, 0xb3, 0xbe, 0xb5, 0x8e, - 0x7b, 0xa6, 0x96, 0x43, 0x0f, 0x40, 0x73, 0x6c, 0xd2, 0x3d, 0xed, 0x58, 0x36, 0x19, 0x5a, 0xe7, - 0xa6, 0x73, 0x31, 0xd4, 0x54, 0x8e, 0x9e, 0x0e, 0x7b, 0x5d, 0x62, 0xbe, 0xeb, 0x9a, 0xa6, 0x31, - 0x20, 0xe7, 0x9d, 0x77, 0x5a, 0x1e, 0x35, 0xe0, 0x81, 0x65, 0x0f, 0x2e, 0x4e, 0x4e, 0xac, 0xae, - 0x65, 0xda, 0x43, 0x72, 0xdc, 0xe9, 0x75, 0xec, 0xae, 0xa9, 0x6d, 0xa3, 0x3d, 0x40, 0x96, 0xdd, - 0x75, 0xce, 0xfb, 0x3d, 0x73, 0x68, 0x12, 0x29, 0xdd, 0x1d, 0x54, 0x87, 0x9a, 0xf0, 0xd3, 0x31, - 0x0c, 0x72, 0xd2, 0xb1, 0x7a, 0xa6, 0xa1, 0x15, 0x78, 0x26, 0x29, 0x63, 0x40, 0x0c, 0x6b, 0xd0, - 0x39, 0xe6, 0x70, 0x91, 0xc7, 0xb4, 0xec, 0x37, 0x8e, 0xd5, 0x35, 0x49, 0x97, 0xbb, 0xe5, 0x28, - 0x70, 0xb2, 0x44, 0x2f, 0x6c, 0xc3, 0xc4, 0xfd, 0x8e, 0x65, 0x68, 0x25, 0xb4, 0x0f, 0x0f, 0x25, - 0x6c, 0xbe, 0xeb, 0x5b, 0xf8, 0x7b, 0x32, 0x74, 0x1c, 0x32, 0x70, 0x1c, 0x5b, 0x2b, 0x67, 0x3d, - 0xf1, 0x6a, 0x9d, 0xbe, 0x69, 0x6b, 0x15, 0xf4, 0x10, 0xea, 0xe7, 0xfd, 0x3e, 0x91, 0x16, 0x59, - 0x6c, 0x95, 0xd3, 0x3b, 0x86, 0x81, 0xcd, 0xc1, 0x80, 0x9c, 0x5b, 0x83, 0xf3, 0xce, 0xb0, 0x7b, - 0xaa, 0xd5, 0x78, 0x49, 0x03, 0x73, 0x48, 0x86, 0xce, 0xb0, 0xd3, 0x5b, 0xe1, 0x1a, 0x4f, 0x68, - 0x85, 0xf3, 0xa0, 0x3d, 0xe7, 0xad, 0xb6, 0xcb, 0x27, 0x9c, 0xc3, 0xce, 0x9b, 0x34, 0x45, 0xc4, - 0x6b, 0x4f, 0xdb, 0x23, 0x63, 0x6a, 0x75, 0x0e, 0x5a, 0xf6, 0x9b, 0x4e, 0xcf, 0x32, 0xc8, 0x99, - 0xf9, 0xbd, 0x58, 0xfa, 0x0f, 0x38, 0x98, 0x64, 0x46, 0xfa, 0xd8, 0xf9, 0x96, 0x27, 0xa2, 0x7d, - 0x8e, 0x10, 0x54, 0xbb, 0x16, 0xee, 0x5e, 0xf4, 0x3a, 0x98, 0x60, 0xe7, 0x62, 0x68, 0x6a, 0x7b, - 0x47, 0xff, 0xc9, 0xc3, 0xb6, 0x38, 0xa8, 0x22, 0xf4, 0x8a, 0xeb, 0x7f, 0xf9, 0x2d, 0x84, 0x1e, - 0x7f, 0xf2, 0x1b, 0xa9, 0x29, 0x2f, 0xb3, 0x29, 0xfc, 0x42, 0x41, 0x7f, 0x82, 0x72, 0xf6, 0xfb, - 0x02, 0x65, 0xf7, 0xf6, 0x0d, 0x1f, 0x1e, 0x1b, 0x3c, 0x9c, 0x81, 0x66, 0xc6, 0xcc, 0x9b, 0xb9, - 0x8c, 0xca, 0xef, 0x05, 0xd4, 0xcc, 0x78, 0xb9, 0xf5, 0x11, 0xd2, 0xdc, 0xdf, 0x68, 0x4b, 0x4f, - 0xe5, 0x5e, 0x52, 0x4e, 0x7a, 0x63, 0xbf, 0x53, 0xce, 0xfa, 0x67, 0x42, 0xf3, 0x8b, 0xfb, 0xcc, - 0xa9, 0xb7, 0x31, 0xd4, 0x37, 0x5c, 0xc2, 0xd1, 0x57, 0xd9, 0x0c, 0xee, 0xbd, 0xc2, 0x37, 0x9f, - 0xfc, 0x10, 0x6d, 0x15, 0x65, 0xc3, 0x6d, 0x7d, 0x2d, 0xca, 0xfd, 0x77, 0xfd, 0xb5, 0x28, 0x9f, - 0xba, 0xf4, 0xbf, 0x07, 0xed, 0xf6, 0xe5, 0x0e, 0xe9, 0xb7, 0xc7, 0xde, 0xbd, 0x65, 0x36, 0x7f, - 0xf6, 0x49, 0x4e, 0xea, 0xdc, 0x02, 0x58, 0x5d, 0x91, 0xd0, 0xa3, 0xcc, 0x90, 0x3b, 0x57, 0xbc, - 0xe6, 0xe3, 0x7b, 0xac, 0xa9, 0xab, 0x21, 0xd4, 0x37, 0xdc, 0x99, 0xd6, 0x66, 0xe3, 0xfe, 0x3b, - 0x55, 0xf3, 0xc1, 0xa6, 0xab, 0xc5, 0x0b, 0xe5, 0xf8, 0xeb, 0x3f, 0x1f, 0x5e, 0x79, 0x6c, 0xba, - 0xb8, 0x3c, 0x18, 0x85, 0xb3, 0x43, 0xdf, 0xbb, 0x9a, 0xb2, 0xc0, 0x0b, 0xae, 0x02, 0xca, 0x3e, - 0x86, 0xd1, 0xf5, 0xa1, 0x1f, 0x8c, 0x0f, 0x85, 0x2e, 0x0f, 0x97, 0xc3, 0x2f, 0xb7, 0xc5, 0x6f, - 0x47, 0xbf, 0xfa, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x74, 0xbe, 0x98, 0x4d, 0x6b, 0x12, 0x00, - 0x00, + // 2039 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x58, 0xcd, 0x76, 0xdb, 0xc6, + 0x15, 0x0e, 0x44, 0x50, 0x22, 0x2f, 0xff, 0xa0, 0xa1, 0x23, 0xb3, 0x54, 0x9c, 0x32, 0x68, 0x63, + 0xf3, 0xb8, 0xae, 0xe4, 0xa8, 0x3d, 0xad, 0x4f, 0xdb, 0xb8, 0xa5, 0x08, 0x28, 0x82, 0x45, 0x01, + 0xcc, 0x10, 0xb2, 0x9d, 0x7a, 0x31, 0x07, 0x22, 0x87, 0x22, 0x2a, 0xfc, 0xb0, 0xc0, 0xd0, 0x8e, + 0x96, 0xdd, 0x76, 0xdf, 0x67, 0xe8, 0x4b, 0xf4, 0x5d, 0xba, 0xed, 0xae, 0xbb, 0xae, 0x7b, 0x66, + 0x00, 0x90, 0xa0, 0x44, 0x39, 0xd9, 0xd8, 0x9c, 0xef, 0x7e, 0x73, 0xe7, 0xde, 0xb9, 0xdf, 0x9d, + 0x19, 0x08, 0xf6, 0xa2, 0x70, 0xc1, 0x68, 0x14, 0xcd, 0xc7, 0x87, 0xc9, 0xaf, 0x83, 0x79, 0x14, + 0xb2, 0x10, 0x95, 0x97, 0x78, 0xbb, 0x1c, 0xcd, 0xc7, 0x09, 0xaa, 0xfe, 0x63, 0x1b, 0xd0, 0x88, + 0x06, 0x93, 0xa1, 0x73, 0xe3, 0xd3, 0x80, 0x61, 0xfa, 0xd7, 0x05, 0x8d, 0x19, 0x42, 0x20, 0x4f, + 0x68, 0xcc, 0x5a, 0x52, 0x47, 0xea, 0x56, 0xb1, 0xf8, 0x8d, 0x14, 0x28, 0x38, 0x3e, 0x6b, 0x6d, + 0x75, 0xa4, 0x6e, 0x01, 0xf3, 0x9f, 0xe8, 0x27, 0x50, 0x72, 0x7c, 0x46, 0xfc, 0xd8, 0x61, 0xad, + 0xaa, 0x80, 0x77, 0x1c, 0x9f, 0x9d, 0xc7, 0x0e, 0x43, 0x5f, 0x40, 0x75, 0x9e, 0xb8, 0x24, 0x33, + 0x27, 0x9e, 0xb5, 0x0a, 0xc2, 0x51, 0x25, 0xc5, 0x4e, 0x9d, 0x78, 0x86, 0xba, 0xa0, 0x4c, 0xdd, + 0xc0, 0xf1, 0xc8, 0xd8, 0x63, 0xef, 0xc9, 0x84, 0x7a, 0xcc, 0x69, 0xc9, 0x1d, 0xa9, 0x5b, 0xc4, + 0x75, 0x81, 0xf7, 0x3d, 0xf6, 0x5e, 0xe3, 0x28, 0x7a, 0x02, 0x8d, 0xcc, 0x59, 0x94, 0x04, 0xd8, + 0x2a, 0x76, 0xa4, 0x6e, 0x19, 0xd7, 0xe7, 0xeb, 0x61, 0x3f, 0x81, 0x06, 0x73, 0x7d, 0x1a, 0x2e, + 0x18, 0x89, 0xe9, 0x38, 0x0c, 0x26, 0x71, 0x6b, 0x3b, 0xf1, 0x98, 0xc2, 0xa3, 0x04, 0x45, 0x2a, + 0xd4, 0xa6, 0x94, 0x12, 0xcf, 0xf5, 0x5d, 0x46, 0x78, 0xf8, 0x3b, 0x22, 0xfc, 0xca, 0x94, 0xd2, + 0x01, 0xc7, 0x46, 0x0e, 0x43, 0x3f, 0x87, 0xfa, 0x8a, 0x23, 0x72, 0xac, 0x09, 0x52, 0x35, 0x23, + 0x89, 0x44, 0x9f, 0x81, 0x12, 0x2e, 0xd8, 0x55, 0xe8, 0x06, 0x57, 0x64, 0x3c, 0x73, 0x02, 0xe2, + 0x4e, 0x5a, 0xa5, 0x8e, 0xd4, 0x95, 0x8f, 0xb7, 0x9e, 0x4b, 0xb8, 0x9e, 0xd9, 0xfa, 0x33, 0x27, + 0x30, 0x26, 0xe8, 0x31, 0x34, 0x3c, 0x27, 0x66, 0x64, 0x16, 0xce, 0xc9, 0x7c, 0x71, 0x79, 0x4d, + 0x6f, 0x5a, 0x75, 0xb1, 0x33, 0x35, 0x0e, 0x9f, 0x86, 0xf3, 0xa1, 0x00, 0xd1, 0x23, 0x00, 0xb1, + 0x2b, 0x62, 0xf1, 0x56, 0x59, 0xe4, 0x50, 0xe6, 0x88, 0x58, 0x18, 0x7d, 0x05, 0x15, 0x51, 0x4d, + 0x32, 0x73, 0x03, 0x16, 0xb7, 0xa0, 0x53, 0xe8, 0x56, 0x8e, 0x94, 0x03, 0x2f, 0xe0, 0x85, 0xc5, + 0xdc, 0x72, 0xea, 0x06, 0x0c, 0x43, 0x94, 0xfd, 0x8c, 0xd1, 0x04, 0x9a, 0xbc, 0x8a, 0x64, 0xbc, + 0x88, 0x59, 0xe8, 0x93, 0x88, 0x8e, 0xc3, 0x68, 0x12, 0xb7, 0x2a, 0x62, 0xea, 0xaf, 0x0f, 0x96, + 0xe2, 0x38, 0xb8, 0xab, 0x86, 0x03, 0x8d, 0xc6, 0xac, 0x2f, 0xe6, 0xe1, 0x64, 0x9a, 0x1e, 0xb0, + 0xe8, 0x06, 0xef, 0x4e, 0x6e, 0xe3, 0xe8, 0x19, 0x20, 0xc7, 0xf3, 0xc2, 0x0f, 0x24, 0xa6, 0xde, + 0x94, 0xa4, 0xd5, 0x69, 0x35, 0x3a, 0x52, 0xb7, 0x84, 0x15, 0x61, 0x19, 0x51, 0x6f, 0x9a, 0xba, + 0x47, 0xbf, 0x81, 0x9a, 0x88, 0x69, 0x4a, 0x1d, 0xb6, 0x88, 0x68, 0xdc, 0x52, 0x3a, 0x85, 0x6e, + 0xfd, 0x68, 0x37, 0x4d, 0xe4, 0x24, 0x81, 0x8f, 0x5d, 0x86, 0xab, 0x9c, 0x97, 0x8e, 0x63, 0xb4, + 0x0f, 0x65, 0xdf, 0xf9, 0x9e, 0xcc, 0x98, 0x37, 0x8e, 0x5b, 0xbb, 0x1d, 0xa9, 0x5b, 0xc3, 0x25, + 0xdf, 0xf9, 0xfe, 0x94, 0x8f, 0xdb, 0x1a, 0xec, 0x6d, 0x8e, 0x97, 0x0b, 0x98, 0x6f, 0x38, 0xd7, + 0xb4, 0x8c, 0xf9, 0x4f, 0xf4, 0x00, 0x8a, 0xef, 0x1d, 0x6f, 0x41, 0x85, 0xa8, 0xab, 0x38, 0x19, + 0xfc, 0x6e, 0xeb, 0x85, 0xa4, 0xbe, 0x80, 0xa6, 0x1d, 0x39, 0xe3, 0xeb, 0x5b, 0x7d, 0x71, 0x5b, + 0xd6, 0xd2, 0x1d, 0x59, 0xab, 0x2f, 0xa1, 0x21, 0x2a, 0x70, 0x42, 0xe9, 0xc7, 0xba, 0xe9, 0x21, + 0xf0, 0x5e, 0x11, 0xda, 0x4b, 0x3a, 0x6a, 0xdb, 0xf1, 0xb9, 0xec, 0xd4, 0x09, 0x28, 0xab, 0xf9, + 0xf1, 0x3c, 0x0c, 0x62, 0xca, 0x5b, 0x85, 0x17, 0x88, 0x6b, 0x8c, 0x4b, 0x52, 0x88, 0x51, 0x12, + 0xb3, 0xea, 0x29, 0x7e, 0x42, 0xa9, 0x90, 0xe3, 0xe3, 0xa4, 0x03, 0x88, 0x17, 0x8e, 0xaf, 0x79, + 0x4f, 0x39, 0x37, 0xa9, 0xfb, 0x1a, 0x87, 0x07, 0xe1, 0xf8, 0x5a, 0xe3, 0xa0, 0xfa, 0x2e, 0x69, + 0x7b, 0x3b, 0x14, 0x6b, 0xfd, 0xf8, 0xf4, 0x90, 0x0a, 0x45, 0xa1, 0x15, 0xe1, 0xb6, 0x72, 0x54, + 0xcd, 0x8b, 0x0e, 0x27, 0x26, 0xf5, 0x1d, 0x34, 0xd7, 0x9c, 0xa7, 0x59, 0xb4, 0xa1, 0x34, 0x8f, + 0xa8, 0xeb, 0x3b, 0x57, 0x34, 0xf5, 0xbc, 0x1c, 0xa3, 0x2e, 0xec, 0x4c, 0x1d, 0xd7, 0x5b, 0x44, + 0x99, 0xe3, 0x7a, 0x26, 0x82, 0x04, 0xc5, 0x99, 0x59, 0xfd, 0x0c, 0xda, 0x98, 0xc6, 0x94, 0x9d, + 0xbb, 0x71, 0xec, 0x86, 0x41, 0x3f, 0x0c, 0x58, 0x14, 0x7a, 0x69, 0x06, 0xea, 0x23, 0xd8, 0xdf, + 0x68, 0x4d, 0x42, 0xe0, 0x93, 0xbf, 0x5d, 0xd0, 0xe8, 0x66, 0xf3, 0xe4, 0x6f, 0x61, 0x7f, 0xa3, + 0x35, 0x8d, 0xff, 0x19, 0x14, 0xe7, 0x8e, 0x1b, 0xc5, 0xad, 0x2d, 0xd1, 0x34, 0x7b, 0xb9, 0xa6, + 0x19, 0x3a, 0x6e, 0x74, 0xea, 0xc6, 0x2c, 0x8c, 0x6e, 0x70, 0x42, 0x7a, 0x25, 0x97, 0x24, 0x65, + 0x4b, 0xfd, 0xbb, 0x04, 0x95, 0x9c, 0x91, 0x4b, 0x37, 0x08, 0x27, 0x94, 0x4c, 0xa3, 0xd0, 0xcf, + 0x36, 0x81, 0x03, 0x27, 0x51, 0xe8, 0x73, 0x4d, 0x08, 0x23, 0x0b, 0x53, 0x41, 0x6e, 0xf3, 0xa1, + 0x1d, 0xa2, 0x5f, 0xc2, 0xce, 0x2c, 0x71, 0x20, 0x0e, 0xaa, 0xca, 0x51, 0xf3, 0xd6, 0xda, 0x9a, + 0xc3, 0x1c, 0x9c, 0x71, 0x5e, 0xc9, 0xa5, 0x82, 0x22, 0xbf, 0x92, 0x4b, 0xb2, 0x52, 0x7c, 0x25, + 0x97, 0x8a, 0xca, 0xf6, 0x2b, 0xb9, 0xb4, 0xad, 0xec, 0xa8, 0xff, 0x91, 0xa0, 0x94, 0xb1, 0x79, + 0x24, 0x7c, 0x4b, 0x09, 0xd7, 0x45, 0x2a, 0xa6, 0x12, 0x07, 0x6c, 0xd7, 0xa7, 0xa8, 0x03, 0x55, + 0x61, 0x5c, 0x97, 0x28, 0x70, 0xac, 0x27, 0x64, 0x2a, 0x4e, 0xd0, 0x8c, 0x21, 0xf4, 0x28, 0xa7, + 0x27, 0x68, 0x42, 0xc9, 0x2e, 0x81, 0x78, 0x31, 0x1e, 0xd3, 0x38, 0x4e, 0x56, 0x29, 0x26, 0x94, + 0x14, 0x13, 0x0b, 0x3d, 0x86, 0x46, 0x46, 0xc9, 0xd6, 0xda, 0x4e, 0xf4, 0x9a, 0xc2, 0xe9, 0x72, + 0x5d, 0x50, 0xf2, 0x3c, 0x7f, 0x75, 0x66, 0xd7, 0x57, 0x44, 0xbe, 0x68, 0x92, 0xbc, 0xfa, 0x17, + 0x78, 0x28, 0x4a, 0x39, 0x8c, 0xc2, 0x4b, 0xe7, 0xd2, 0xf5, 0x5c, 0x76, 0x93, 0x89, 0x9c, 0x27, + 0x1e, 0x85, 0x3e, 0xe1, 0x7b, 0x9b, 0x95, 0x80, 0x03, 0x66, 0x38, 0xa1, 0xbc, 0x04, 0x2c, 0x4c, + 0x4c, 0x69, 0x09, 0x58, 0x28, 0x0c, 0xf9, 0xbb, 0xae, 0xb0, 0x76, 0xd7, 0xa9, 0xd7, 0xd0, 0xba, + 0xbb, 0x56, 0xaa, 0x99, 0x0e, 0x54, 0xe6, 0x2b, 0x58, 0x2c, 0x27, 0xe1, 0x3c, 0x94, 0xaf, 0xed, + 0xd6, 0x0f, 0xd7, 0x56, 0xfd, 0xa7, 0x04, 0xbb, 0xc7, 0x0b, 0xd7, 0x9b, 0xac, 0x35, 0x6e, 0x3e, + 0x3a, 0x69, 0xfd, 0x26, 0xde, 0x74, 0xcd, 0x6e, 0x6d, 0xbc, 0x66, 0x37, 0x5d, 0x65, 0x85, 0x7b, + 0xaf, 0xb2, 0x9f, 0x42, 0x65, 0x75, 0x8b, 0xc5, 0x2d, 0xb9, 0x53, 0xe8, 0x56, 0x31, 0xcc, 0xb2, + 0x2b, 0x2c, 0x56, 0x5f, 0x00, 0xca, 0x07, 0x9a, 0x6e, 0xc8, 0xf2, 0xfc, 0x90, 0xee, 0x3f, 0x3f, + 0x3e, 0x83, 0xf6, 0x68, 0x71, 0x19, 0x8f, 0x23, 0xf7, 0x92, 0xf2, 0x43, 0x5d, 0x7f, 0x4f, 0x03, + 0x16, 0x67, 0x5d, 0xfa, 0x3f, 0x19, 0xca, 0x4b, 0x14, 0x1d, 0x40, 0xd3, 0x0d, 0xc6, 0xa1, 0x9f, + 0x05, 0x1d, 0x50, 0x8f, 0xc7, 0x9d, 0x1c, 0xf2, 0xbb, 0x99, 0xa9, 0x9f, 0x58, 0x8c, 0x09, 0xe7, + 0xaf, 0x25, 0x99, 0xf2, 0xb7, 0x12, 0x7e, 0x3e, 0xc7, 0x84, 0xdf, 0x05, 0x65, 0xe9, 0x9f, 0x5f, + 0x38, 0xcb, 0x4d, 0xc1, 0xf5, 0x0c, 0xe7, 0xc1, 0x24, 0xcc, 0xa5, 0xe7, 0x8c, 0x29, 0x27, 0xcc, + 0x0c, 0x4f, 0x99, 0x5f, 0x40, 0x95, 0xf7, 0x43, 0xcc, 0x1c, 0x7f, 0x4e, 0x82, 0x58, 0xf4, 0x85, + 0x8c, 0x2b, 0x4b, 0xcc, 0x8c, 0xd1, 0xd7, 0x00, 0x94, 0xe7, 0x47, 0xd8, 0xcd, 0x9c, 0x8a, 0x96, + 0xa8, 0x1f, 0x7d, 0x9e, 0x13, 0xc6, 0x72, 0x03, 0x0e, 0xc4, 0xbf, 0xf6, 0xcd, 0x9c, 0xe2, 0x32, + 0xcd, 0x7e, 0xa2, 0x97, 0x50, 0x9b, 0x86, 0xd1, 0x07, 0x27, 0x9a, 0x10, 0x01, 0xa6, 0xc7, 0xc6, + 0xc3, 0x9c, 0x87, 0x93, 0xc4, 0x2e, 0xa6, 0x9f, 0x7e, 0x82, 0xab, 0xd3, 0xdc, 0x18, 0x9d, 0x01, + 0xca, 0xe6, 0x8b, 0x2e, 0x4f, 0x9c, 0x94, 0x84, 0x93, 0xfd, 0xbb, 0x4e, 0xf8, 0x21, 0x9d, 0x39, + 0x52, 0xa6, 0xb7, 0x30, 0xf4, 0x7b, 0xa8, 0xc6, 0x94, 0x31, 0x8f, 0xa6, 0x6e, 0xca, 0xc2, 0xcd, + 0xde, 0xda, 0x9b, 0x83, 0x9b, 0x33, 0x0f, 0x95, 0x78, 0x35, 0x44, 0xc7, 0xd0, 0xf0, 0xdc, 0xe0, + 0x3a, 0x1f, 0x06, 0x88, 0xf9, 0xad, 0xdc, 0xfc, 0x81, 0x1b, 0x5c, 0xe7, 0x63, 0xa8, 0x79, 0x79, + 0x40, 0xfd, 0x03, 0x94, 0x97, 0xbb, 0x84, 0x2a, 0xb0, 0x73, 0x61, 0x9e, 0x99, 0xd6, 0x1b, 0x53, + 0xf9, 0x04, 0x95, 0x40, 0x1e, 0xe9, 0xa6, 0xa6, 0x48, 0x1c, 0xc6, 0x7a, 0x5f, 0x37, 0x5e, 0xeb, + 0xca, 0x16, 0x1f, 0x9c, 0x58, 0xf8, 0x4d, 0x0f, 0x6b, 0x4a, 0xe1, 0x78, 0x07, 0x8a, 0x62, 0x5d, + 0xf5, 0x5f, 0x12, 0x94, 0x44, 0x05, 0x83, 0x69, 0x88, 0x7e, 0x01, 0x4b, 0x71, 0x89, 0xc3, 0x8d, + 0x5f, 0xb8, 0x42, 0x75, 0x35, 0xbc, 0x14, 0x8c, 0x9d, 0xe2, 0x9c, 0xbc, 0x94, 0xc6, 0x92, 0xbc, + 0x95, 0x90, 0x33, 0xc3, 0x92, 0xfc, 0x34, 0xe7, 0x79, 0xed, 0xc8, 0x91, 0x71, 0x23, 0x33, 0x64, + 0x27, 0xec, 0xd3, 0x9c, 0xe3, 0xb5, 0x93, 0x58, 0xc6, 0x8d, 0xcc, 0x90, 0x72, 0xd5, 0xdf, 0x42, + 0x35, 0x5f, 0x73, 0xf4, 0x04, 0x64, 0x37, 0x98, 0x86, 0x69, 0x23, 0x36, 0x6f, 0x89, 0x8b, 0x27, + 0x89, 0x05, 0x41, 0x45, 0xa0, 0xdc, 0xae, 0xb3, 0x5a, 0x83, 0x4a, 0xae, 0x68, 0xea, 0xbf, 0x25, + 0xa8, 0xad, 0x15, 0xe1, 0x47, 0x7b, 0x47, 0x5f, 0x43, 0xf5, 0x83, 0x1b, 0x51, 0x92, 0xbf, 0xfe, + 0xeb, 0x47, 0xed, 0xf5, 0xeb, 0x3f, 0xfb, 0xbf, 0x1f, 0x4e, 0x28, 0xae, 0x70, 0x7e, 0x0a, 0xa0, + 0x3f, 0x42, 0x3d, 0x9d, 0x49, 0x26, 0x94, 0x39, 0xae, 0x27, 0xb6, 0xaa, 0xbe, 0x26, 0x8f, 0x94, + 0xab, 0x09, 0x3b, 0xae, 0x4d, 0xf3, 0x43, 0xf4, 0xe5, 0xca, 0x41, 0xcc, 0x22, 0x37, 0xb8, 0x12, + 0xfb, 0x57, 0x5e, 0xd2, 0x46, 0x02, 0x7c, 0xfa, 0x37, 0x19, 0x6a, 0x6b, 0x7e, 0xd6, 0x85, 0x54, + 0x83, 0xb2, 0x69, 0x11, 0x4d, 0xb7, 0x7b, 0xc6, 0x40, 0x91, 0x90, 0x02, 0x55, 0xcb, 0x34, 0x2c, + 0x93, 0x68, 0x7a, 0xdf, 0xd2, 0xb8, 0xa4, 0x3e, 0x85, 0xdd, 0x81, 0x61, 0x9e, 0x11, 0xd3, 0xb2, + 0x89, 0x3e, 0x30, 0xbe, 0x31, 0x8e, 0x07, 0xba, 0x52, 0x40, 0x0f, 0x40, 0xb1, 0x4c, 0xd2, 0x3f, + 0xed, 0x19, 0x26, 0xb1, 0x8d, 0x73, 0xdd, 0xba, 0xb0, 0x15, 0x99, 0xa3, 0xa7, 0xf6, 0xa0, 0x4f, + 0xf4, 0xb7, 0x7d, 0x5d, 0xd7, 0x46, 0xe4, 0xbc, 0xf7, 0x56, 0x29, 0xa2, 0x16, 0x3c, 0x30, 0xcc, + 0xd1, 0xc5, 0xc9, 0x89, 0xd1, 0x37, 0x74, 0xd3, 0x26, 0xc7, 0xbd, 0x41, 0xcf, 0xec, 0xeb, 0xca, + 0x36, 0xda, 0x03, 0x64, 0x98, 0x7d, 0xeb, 0x7c, 0x38, 0xd0, 0x6d, 0x9d, 0x64, 0xd2, 0xdd, 0x41, + 0x4d, 0x68, 0x08, 0x3f, 0x3d, 0x4d, 0x23, 0x27, 0x3d, 0x63, 0xa0, 0x6b, 0x4a, 0x89, 0x47, 0x92, + 0x32, 0x46, 0x44, 0x33, 0x46, 0xbd, 0x63, 0x0e, 0x97, 0xf9, 0x9a, 0x86, 0xf9, 0xda, 0x32, 0xfa, + 0x3a, 0xe9, 0x73, 0xb7, 0x1c, 0x05, 0x4e, 0xce, 0xd0, 0x0b, 0x53, 0xd3, 0xf1, 0xb0, 0x67, 0x68, + 0x4a, 0x05, 0xed, 0xc3, 0xc3, 0x0c, 0xd6, 0xdf, 0x0e, 0x0d, 0xfc, 0x1d, 0xb1, 0x2d, 0x8b, 0x8c, + 0x2c, 0xcb, 0x54, 0xaa, 0x79, 0x4f, 0x3c, 0x5b, 0x6b, 0xa8, 0x9b, 0x4a, 0x0d, 0x3d, 0x84, 0xe6, + 0xf9, 0x70, 0x48, 0x32, 0x4b, 0x96, 0x6c, 0x9d, 0xd3, 0x7b, 0x9a, 0x86, 0xf5, 0xd1, 0x88, 0x9c, + 0x1b, 0xa3, 0xf3, 0x9e, 0xdd, 0x3f, 0x55, 0x1a, 0x3c, 0xa5, 0x91, 0x6e, 0x13, 0xdb, 0xb2, 0x7b, + 0x83, 0x15, 0xae, 0xf0, 0x80, 0x56, 0x38, 0x5f, 0x74, 0x60, 0xbd, 0x51, 0x76, 0xf9, 0x86, 0x73, + 0xd8, 0x7a, 0x9d, 0x86, 0x88, 0x78, 0xee, 0x69, 0x79, 0xb2, 0x35, 0x95, 0x26, 0x07, 0x0d, 0xf3, + 0x75, 0x6f, 0x60, 0x68, 0xe4, 0x4c, 0xff, 0x4e, 0xb4, 0xfe, 0x03, 0x0e, 0x26, 0x91, 0x91, 0x21, + 0xb6, 0xbe, 0xe1, 0x81, 0x28, 0x9f, 0x22, 0x04, 0xf5, 0xbe, 0x81, 0xfb, 0x17, 0x83, 0x1e, 0x26, + 0xd8, 0xba, 0xb0, 0x75, 0x65, 0xef, 0xe8, 0xbf, 0x45, 0xd8, 0x16, 0x17, 0x55, 0x84, 0x5e, 0x72, + 0xfd, 0x2f, 0x3f, 0x94, 0xd0, 0xa3, 0x8f, 0x7e, 0x40, 0xb5, 0xb3, 0xc7, 0x6c, 0x0a, 0x3f, 0x97, + 0xd0, 0x9f, 0xa0, 0x9a, 0xff, 0xbe, 0x40, 0xf9, 0xb3, 0x7d, 0xc3, 0x87, 0xc7, 0x06, 0x0f, 0x67, + 0xa0, 0xe8, 0x31, 0x73, 0x7d, 0x87, 0xd1, 0xec, 0x7b, 0x01, 0xb5, 0x73, 0x5e, 0x6e, 0x7d, 0x84, + 0xb4, 0xf7, 0x37, 0xda, 0xd2, 0x5b, 0x79, 0x90, 0xa4, 0x93, 0xbe, 0xd8, 0xef, 0xa4, 0xb3, 0xfe, + 0x99, 0xd0, 0xfe, 0xfc, 0x3e, 0x73, 0xea, 0x6d, 0x02, 0xcd, 0x0d, 0x8f, 0x70, 0xf4, 0x65, 0x3e, + 0x82, 0x7b, 0x9f, 0xf0, 0xed, 0xc7, 0x3f, 0x44, 0x5b, 0xad, 0xb2, 0xe1, 0xb5, 0xbe, 0xb6, 0xca, + 0xfd, 0x6f, 0xfd, 0xb5, 0x55, 0x3e, 0xf6, 0xe8, 0x7f, 0x07, 0xca, 0xed, 0xc7, 0x1d, 0x52, 0x6f, + 0xcf, 0xbd, 0xfb, 0xca, 0x6c, 0xff, 0xec, 0xa3, 0x9c, 0xd4, 0xb9, 0x01, 0xb0, 0x7a, 0x22, 0xa1, + 0xcf, 0x72, 0x53, 0xee, 0x3c, 0xf1, 0xda, 0x8f, 0xee, 0xb1, 0xa6, 0xae, 0x6c, 0x68, 0x6e, 0x78, + 0x33, 0xad, 0xed, 0xc6, 0xfd, 0x6f, 0xaa, 0xf6, 0x83, 0x4d, 0x4f, 0x8b, 0xe7, 0xd2, 0xf1, 0x57, + 0x7f, 0x3e, 0xbc, 0x72, 0xd9, 0x6c, 0x71, 0x79, 0x30, 0x0e, 0xfd, 0x43, 0xcf, 0xbd, 0x9a, 0xb1, + 0xc0, 0x0d, 0xae, 0x02, 0xca, 0x3e, 0x84, 0xd1, 0xf5, 0xa1, 0x17, 0x4c, 0x0e, 0x85, 0x2e, 0x0f, + 0x97, 0xd3, 0x2f, 0xb7, 0xc5, 0x1f, 0x96, 0x7e, 0xf5, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x5f, + 0x5a, 0x7c, 0x81, 0x88, 0x12, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/lnrpc/routerrpc/router.proto b/lnrpc/routerrpc/router.proto index 43cffc583..25e47d999 100644 --- a/lnrpc/routerrpc/router.proto +++ b/lnrpc/routerrpc/router.proto @@ -114,6 +114,12 @@ message SendPaymentRequest { fallback. */ repeated lnrpc.FeatureBit dest_features = 16; + + /** + The maximum number of partial payments that may be use to complete the full + amount. + */ + uint32 max_htlcs = 17; } message TrackPaymentRequest { diff --git a/lnrpc/routerrpc/router_backend.go b/lnrpc/routerrpc/router_backend.go index 0a64161bb..d4d75edd3 100644 --- a/lnrpc/routerrpc/router_backend.go +++ b/lnrpc/routerrpc/router_backend.go @@ -547,6 +547,14 @@ func (r *RouterBackend) extractIntentFromSendRequest( } payIntent.CltvLimit = cltvLimit + // Take max htlcs from the request. Map zero to one for backwards + // compatibility. + maxHtlcs := rpcPayReq.MaxHtlcs + if maxHtlcs == 0 { + maxHtlcs = 1 + } + payIntent.MaxHtlcs = maxHtlcs + // Take fee limit from request. payIntent.FeeLimit, err = lnrpc.UnmarshallAmt( rpcPayReq.FeeLimitSat, rpcPayReq.FeeLimitMsat, diff --git a/routing/integrated_routing_context_test.go b/routing/integrated_routing_context_test.go index 418b380ca..4ab395e71 100644 --- a/routing/integrated_routing_context_test.go +++ b/routing/integrated_routing_context_test.go @@ -1,7 +1,9 @@ package routing import ( + "fmt" "io/ioutil" + "math" "os" "testing" "time" @@ -64,6 +66,7 @@ func newIntegratedRoutingContext(t *testing.T) *integratedRoutingContext { pathFindingCfg: PathFindingConfig{ PaymentAttemptPenalty: 1000, + MinProbability: 0.01, }, source: source, @@ -79,9 +82,15 @@ type htlcAttempt struct { success bool } +func (h htlcAttempt) String() string { + return fmt.Sprintf("success=%v, route=%v", h.success, h.route) +} + // testPayment launches a test payment and asserts that it is completed after // the expected number of attempts. -func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) { +func (c *integratedRoutingContext) testPayment(maxHtlcs uint32) ([]htlcAttempt, + error) { + var ( nextPid uint64 attempts []htlcAttempt @@ -119,10 +128,16 @@ func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) { return bandwidthHints, nil } + var paymentAddr [32]byte payment := LightningPayment{ FinalCLTVDelta: uint16(c.finalExpiry), FeeLimit: lnwire.MaxMilliSatoshi, Target: c.target.pubkey, + PaymentAddr: &paymentAddr, + DestFeatures: lnwire.NewFeatureVector(mppFeatures, nil), + Amount: c.amt, + CltvLimit: math.MaxUint32, + MaxHtlcs: maxHtlcs, } session := &paymentSession{ @@ -134,10 +149,15 @@ func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) { }, pathFindingConfig: c.pathFindingCfg, missionControl: mc, + minShardAmt: lnwire.NewMSatFromSatoshis(5000), } // Now the payment control loop starts. It will keep trying routes until // the payment succeeds. + var ( + amtRemaining = payment.Amount + inFlightHtlcs uint32 + ) for { // Create bandwidth hints based on local channel balances. bandwidthHints := map[uint64]lnwire.MilliSatoshi{} @@ -147,10 +167,10 @@ func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) { // Find a route. route, err := session.RequestRoute( - c.amt, lnwire.MaxMilliSatoshi, 0, 0, + amtRemaining, lnwire.MaxMilliSatoshi, inFlightHtlcs, 0, ) if err != nil { - return nil, err + return attempts, err } // Send out the htlc on the mock graph. @@ -167,21 +187,33 @@ func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) { success: success, }) - // Process the result. + // Process the result. In normal Lightning operations, the + // sender doesn't get an acknowledgement from the recipient that + // the htlc arrived. In integrated routing tests, this + // acknowledgement is available. It is a simplification of + // reality that still allows certain classes of tests to be + // performed. if success { + inFlightHtlcs++ + err := mc.ReportPaymentSuccess(pid, route) if err != nil { c.t.Fatal(err) } - // If the payment is successful, the control loop can be - // broken out of. - break + amtRemaining -= route.ReceiverAmt() + + // If the full amount has been paid, the payment is + // successful and the control loop can be terminated. + if amtRemaining == 0 { + break + } + + // Otherwise try to send the remaining amount. + continue } // Failure, update mission control and retry. - c.t.Logf("fail: %v @ %v\n", htlcResult.failure, htlcResult.failureSource) - finalResult, err := mc.ReportPaymentFail( pid, route, getNodeIndex(route, htlcResult.failureSource), @@ -192,13 +224,10 @@ func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) { } if finalResult != nil { - c.t.Logf("final result: %v\n", finalResult) break } } - c.t.Logf("Payment attempts: %v\n", len(attempts)) - return attempts, nil } diff --git a/routing/integrated_routing_test.go b/routing/integrated_routing_test.go index 1e77ce810..fb5d22547 100644 --- a/routing/integrated_routing_test.go +++ b/routing/integrated_routing_test.go @@ -2,6 +2,9 @@ package routing import ( "testing" + + "github.com/btcsuite/btcutil" + "github.com/lightningnetwork/lnd/lnwire" ) // TestProbabilityExtrapolation tests that probabilities for tried channels are @@ -50,7 +53,7 @@ func TestProbabilityExtrapolation(t *testing.T) { // a specific number of attempts to safe-guard against accidental // modifications anywhere in the chain of components that is involved in // this test. - attempts, err := ctx.testPayment() + attempts, err := ctx.testPayment(1) if err != nil { t.Fatalf("payment failed: %v", err) } @@ -62,7 +65,7 @@ func TestProbabilityExtrapolation(t *testing.T) { // of data from other channels), all ten bad channels will be tried // first before switching to the paid channel. ctx.mcCfg.AprioriWeight = 1 - attempts, err = ctx.testPayment() + attempts, err = ctx.testPayment(1) if err != nil { t.Fatalf("payment failed: %v", err) } @@ -70,3 +73,211 @@ func TestProbabilityExtrapolation(t *testing.T) { t.Fatalf("expected 11 attempts, but needed %v", len(attempts)) } } + +type mppSendTestCase struct { + name string + amt btcutil.Amount + expectedAttempts int + + // expectedSuccesses is a list of htlcs that made it to the receiver, + // regardless of whether the final set became complete or not. + expectedSuccesses []expectedHtlcSuccess + + graph func(g *mockGraph) + expectedFailure bool + maxHtlcs uint32 +} + +const ( + chanSourceIm1 = 13 + chanIm1Target = 32 + chanSourceIm2 = 14 + chanIm2Target = 42 +) + +func onePathGraph(g *mockGraph) { + // Create the following network of nodes: + // source -> intermediate1 -> target + + const im1NodeID = 3 + intermediate1 := newMockNode(im1NodeID) + g.addNode(intermediate1) + + g.addChannel(chanSourceIm1, sourceNodeID, im1NodeID, 200000) + g.addChannel(chanIm1Target, targetNodeID, im1NodeID, 100000) +} + +func twoPathGraph(g *mockGraph) { + // Create the following network of nodes: + // source -> intermediate1 -> target + // source -> intermediate2 -> target + + const im1NodeID = 3 + intermediate1 := newMockNode(im1NodeID) + g.addNode(intermediate1) + + const im2NodeID = 4 + intermediate2 := newMockNode(im2NodeID) + g.addNode(intermediate2) + + g.addChannel(chanSourceIm1, sourceNodeID, im1NodeID, 200000) + g.addChannel(chanSourceIm2, sourceNodeID, im2NodeID, 200000) + g.addChannel(chanIm1Target, targetNodeID, im1NodeID, 100000) + g.addChannel(chanIm2Target, targetNodeID, im2NodeID, 100000) +} + +var mppTestCases = []mppSendTestCase{ + // Test a two-path graph with sufficient liquidity. It is expected that + // pathfinding will try first try to send the full amount via the two + // available routes. When that fails, it will half the amount to 35k sat + // and retry. That attempt reaches the target successfully. Then the + // same route is tried again. Because the channel only had 50k sat, it + // will fail. Finally the second route is tried for 35k and it succeeds + // too. Mpp payment complete. + { + + name: "sufficient inbound", + graph: twoPathGraph, + amt: 70000, + expectedAttempts: 5, + expectedSuccesses: []expectedHtlcSuccess{ + { + amt: 35000, + chans: []uint64{chanSourceIm1, chanIm1Target}, + }, + { + amt: 35000, + chans: []uint64{chanSourceIm2, chanIm2Target}, + }, + }, + maxHtlcs: 1000, + }, + + // Test that a cap on the max htlcs makes it impossible to pay. + { + name: "no splitting", + graph: twoPathGraph, + amt: 70000, + expectedAttempts: 2, + expectedSuccesses: []expectedHtlcSuccess{}, + expectedFailure: true, + maxHtlcs: 1, + }, + + // Test that an attempt is made to split the payment in multiple parts + // that all use the same route if the full amount cannot be sent in a + // single htlc. The sender is effectively probing the receiver's + // incoming channel to see if it has sufficient balance. In this test + // case, the endeavour fails. + { + + name: "one path split", + graph: onePathGraph, + amt: 70000, + expectedAttempts: 7, + expectedSuccesses: []expectedHtlcSuccess{ + { + amt: 35000, + chans: []uint64{chanSourceIm1, chanIm1Target}, + }, + { + amt: 8750, + chans: []uint64{chanSourceIm1, chanIm1Target}, + }, + }, + expectedFailure: true, + maxHtlcs: 1000, + }, +} + +// TestMppSend tests that a payment can be completed using multiple shards. +func TestMppSend(t *testing.T) { + for _, testCase := range mppTestCases { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + testMppSend(t, &testCase) + }) + } +} + +func testMppSend(t *testing.T, testCase *mppSendTestCase) { + ctx := newIntegratedRoutingContext(t) + + g := ctx.graph + testCase.graph(g) + + ctx.amt = lnwire.NewMSatFromSatoshis(testCase.amt) + + attempts, err := ctx.testPayment(testCase.maxHtlcs) + switch { + case err == nil && testCase.expectedFailure: + t.Fatal("expected payment to fail") + case err != nil && !testCase.expectedFailure: + t.Fatal("expected payment to succeed") + } + + if len(attempts) != testCase.expectedAttempts { + t.Fatalf("expected %v attempts, but needed %v", + testCase.expectedAttempts, len(attempts), + ) + } + + assertSuccessAttempts(t, attempts, testCase.expectedSuccesses) +} + +// expectedHtlcSuccess describes an expected successful htlc attempt. +type expectedHtlcSuccess struct { + amt btcutil.Amount + chans []uint64 +} + +// equals matches the expectation with an actual attempt. +func (e *expectedHtlcSuccess) equals(a htlcAttempt) bool { + if a.route.TotalAmount != + lnwire.NewMSatFromSatoshis(e.amt) { + + return false + } + + if len(a.route.Hops) != len(e.chans) { + return false + } + + for i, h := range a.route.Hops { + if h.ChannelID != e.chans[i] { + return false + } + } + + return true +} + +// assertSuccessAttempts asserts that the set of successful htlc attempts +// matches the given expectation. +func assertSuccessAttempts(t *testing.T, attempts []htlcAttempt, + expected []expectedHtlcSuccess) { + + successCount := 0 +loop: + for _, a := range attempts { + if !a.success { + continue + } + + successCount++ + + for _, exp := range expected { + if exp.equals(a) { + continue loop + } + } + + t.Fatalf("htlc success %v not found", a) + } + + if successCount != len(expected) { + t.Fatalf("expected %v successful htlcs, but got %v", + expected, successCount) + } +} diff --git a/routing/pathfind_test.go b/routing/pathfind_test.go index ab54534bd..0683f36ee 100644 --- a/routing/pathfind_test.go +++ b/routing/pathfind_test.go @@ -81,6 +81,12 @@ var ( ), lnwire.Features, ) + mppFeatures = lnwire.NewRawFeatureVector( + lnwire.TLVOnionPayloadOptional, + lnwire.PaymentAddrOptional, + lnwire.MPPOptional, + ) + unknownRequiredFeatures = lnwire.NewFeatureVector( lnwire.NewRawFeatureVector(100), lnwire.Features, ) diff --git a/routing/payment_session.go b/routing/payment_session.go index 9f9c347c9..d06356e7e 100644 --- a/routing/payment_session.go +++ b/routing/payment_session.go @@ -31,6 +31,13 @@ const ( errEmptyPaySession ) +var ( + // DefaultShardMinAmt is the default amount beyond which we won't try to + // further split the payment if no route is found. It is the minimum + // amount that we use as the shard size when splitting. + DefaultShardMinAmt = lnwire.NewMSatFromSatoshis(10000) +) + // Error returns the string representation of the noRouteError func (e noRouteError) Error() string { switch e { @@ -111,6 +118,12 @@ type paymentSession struct { pathFindingConfig PathFindingConfig missionControl MissionController + + // minShardAmt is the amount beyond which we won't try to further split + // the payment if no route is found. If the maximum number of htlcs + // specified in the payment is one, under no circumstances splitting + // will happen and this value remains unused. + minShardAmt lnwire.MilliSatoshi } // RequestRoute returns a route which is likely to be capable for successfully @@ -155,58 +168,87 @@ func (p *paymentSession) RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi, PaymentAddr: p.payment.PaymentAddr, } - // We'll also obtain a set of bandwidthHints from the lower layer for - // each of our outbound channels. This will allow the path finding to - // skip any links that aren't active or just don't have enough bandwidth - // to carry the payment. New bandwidth hints are queried for every new - // path finding attempt, because concurrent payments may change - // balances. - bandwidthHints, err := p.getBandwidthHints() - if err != nil { - return nil, err - } - finalHtlcExpiry := int32(height) + int32(finalCltvDelta) - routingGraph, cleanup, err := p.getRoutingGraph() - if err != nil { - return nil, err + for { + // We'll also obtain a set of bandwidthHints from the lower + // layer for each of our outbound channels. This will allow the + // path finding to skip any links that aren't active or just + // don't have enough bandwidth to carry the payment. New + // bandwidth hints are queried for every new path finding + // attempt, because concurrent payments may change balances. + bandwidthHints, err := p.getBandwidthHints() + if err != nil { + return nil, err + } + + log.Debugf("PaymentSession for %x: trying pathfinding with %v", + p.payment.PaymentHash, maxAmt) + + // Get a routing graph. + routingGraph, cleanup, err := p.getRoutingGraph() + if err != nil { + return nil, err + } + + sourceVertex := routingGraph.sourceNode() + + // Find a route for the current amount. + path, err := p.pathFinder( + &graphParams{ + additionalEdges: p.additionalEdges, + bandwidthHints: bandwidthHints, + graph: routingGraph, + }, + restrictions, &p.pathFindingConfig, + sourceVertex, p.payment.Target, + maxAmt, finalHtlcExpiry, + ) + + // Close routing graph. + cleanup() + + switch { + case err == errNoPathFound: + // No splitting if this is the last shard. + isLastShard := activeShards+1 >= p.payment.MaxHtlcs + if isLastShard { + return nil, errNoPathFound + } + + // This is where the magic happens. If we can't find a + // route, try it for half the amount. + maxAmt /= 2 + + // Put a lower bound on the minimum shard size. + if maxAmt < p.minShardAmt { + return nil, errNoPathFound + } + + // Go pathfinding. + continue + + case err != nil: + return nil, err + } + + // With the next candidate path found, we'll attempt to turn + // this into a route by applying the time-lock and fee + // requirements. + route, err := newRoute( + sourceVertex, path, height, + finalHopParams{ + amt: maxAmt, + totalAmt: p.payment.Amount, + cltvDelta: finalCltvDelta, + records: p.payment.DestCustomRecords, + paymentAddr: p.payment.PaymentAddr, + }, + ) + if err != nil { + return nil, err + } + + return route, err } - defer cleanup() - - sourceVertex := routingGraph.sourceNode() - - path, err := p.pathFinder( - &graphParams{ - additionalEdges: p.additionalEdges, - bandwidthHints: bandwidthHints, - graph: routingGraph, - }, - restrictions, &p.pathFindingConfig, - sourceVertex, p.payment.Target, - maxAmt, finalHtlcExpiry, - ) - if err != nil { - return nil, err - } - - // With the next candidate path found, we'll attempt to turn this into - // a route by applying the time-lock and fee requirements. - route, err := newRoute( - sourceVertex, path, height, - finalHopParams{ - amt: maxAmt, - totalAmt: maxAmt, - cltvDelta: finalCltvDelta, - records: p.payment.DestCustomRecords, - paymentAddr: p.payment.PaymentAddr, - }, - ) - if err != nil { - // TODO(roasbeef): return which edge/vertex didn't work - // out - return nil, err - } - - return route, err } diff --git a/routing/payment_session_source.go b/routing/payment_session_source.go index f3b6607e1..a6016ab85 100644 --- a/routing/payment_session_source.go +++ b/routing/payment_session_source.go @@ -86,6 +86,7 @@ func (m *SessionSource) NewPaymentSession(p *LightningPayment) ( getRoutingGraph: m.getRoutingGraph, pathFindingConfig: m.PathFindingConfig, missionControl: m.MissionControl, + minShardAmt: DefaultShardMinAmt, }, nil } diff --git a/routing/router.go b/routing/router.go index 7d80305b9..d93c38302 100644 --- a/routing/router.go +++ b/routing/router.go @@ -1624,6 +1624,10 @@ type LightningPayment struct { // understand this new onion payload format, then the payment will // fail. DestCustomRecords record.CustomSet + + // MaxHtlcs is the maximum number of partial payments that may be use to + // complete the full amount. + MaxHtlcs uint32 } // SendPayment attempts to send a payment as described within the passed diff --git a/rpcserver.go b/rpcserver.go index 4d7cc1d22..3928eeb11 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -3962,6 +3962,10 @@ func (r *rpcServer) dispatchPaymentIntent( DestCustomRecords: payIntent.destCustomRecords, DestFeatures: payIntent.destFeatures, PaymentAddr: payIntent.paymentAddr, + + // Don't enable multi-part payments on the main rpc. + // Users need to use routerrpc for that. + MaxHtlcs: 1, } preImage, route, routerErr = r.server.chanRouter.SendPayment( From f5c3f930c40f2bac2c7b635c8cc93287377f75c1 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Fri, 27 Mar 2020 14:44:44 +0100 Subject: [PATCH 09/13] lncli: expose max payment htlcs --- cmd/lncli/cmd_pay.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/lncli/cmd_pay.go b/cmd/lncli/cmd_pay.go index 34fb88e9c..426c75166 100644 --- a/cmd/lncli/cmd_pay.go +++ b/cmd/lncli/cmd_pay.go @@ -52,6 +52,13 @@ var ( Usage: "if set, intermediate payment state updates will be " + "displayed", } + + maxHtlcsFlag = cli.UintFlag{ + Name: "max_htlcs", + Usage: "the maximum number of partial payments that may be " + + "used", + Value: 1, + } ) // paymentFlags returns common flags for sendpayment and payinvoice. @@ -88,7 +95,7 @@ func paymentFlags() []cli.Flag { Name: "allow_self_payment", Usage: "allow sending a circular payment to self", }, - dataFlag, showInflightFlag, + dataFlag, showInflightFlag, maxHtlcsFlag, } } @@ -318,6 +325,8 @@ func sendPaymentRequest(ctx *cli.Context, req.AllowSelfPayment = ctx.Bool("allow_self_payment") + req.MaxHtlcs = uint32(ctx.Uint(maxHtlcsFlag.Name)) + // Parse custom data records. data := ctx.String(dataFlag.Name) if data != "" { From 110c99f67699f433d3ad885fd3ffe04c3dc5492b Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Wed, 25 Mar 2020 16:21:58 +0100 Subject: [PATCH 10/13] routing: continue trying after mpp timeout It can happen that the receiver times out some htlcs of the set if it took to long to complete. But because the sender's mission control is now updated, it is worth to keep trying to send those shards again. --- routing/result_interpretation.go | 17 +++-------------- routing/result_interpretation_test.go | 10 ++++++---- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/routing/result_interpretation.go b/routing/result_interpretation.go index 660ce69cc..70cc08f6a 100644 --- a/routing/result_interpretation.go +++ b/routing/result_interpretation.go @@ -215,20 +215,9 @@ func (i *interpretedResult) processPaymentOutcomeFinal( i.finalFailureReason = &reasonIncorrectDetails case *lnwire.FailMPPTimeout: - // TODO(carla): decide how to penalize mpp timeout. In the - // meantime, attribute success to the hops along the route and - // do not penalize the final node. - - i.finalFailureReason = &reasonError - - // If this is a direct payment, take no action. - if n == 1 { - return - } - - // Assign all pairs a success result except the final hop, as - // the payment reached the destination correctly. - i.successPairRange(route, 0, n-2) + // Assign all pairs a success result, as the payment reached the + // destination correctly. Continue the payment process. + i.successPairRange(route, 0, n-1) default: // All other errors are considered terminal if coming from the diff --git a/routing/result_interpretation_test.go b/routing/result_interpretation_test.go index 537869f0b..2fc0a7d90 100644 --- a/routing/result_interpretation_test.go +++ b/routing/result_interpretation_test.go @@ -325,8 +325,10 @@ var resultTestCases = []resultTestCase{ failure: &lnwire.FailMPPTimeout{}, expectedResult: &interpretedResult{ - finalFailureReason: &reasonError, - nodeFailure: nil, + pairResults: map[DirectedNodePair]pairResult{ + getTestPair(0, 1): successPairResult(100), + }, + nodeFailure: nil, }, }, @@ -342,9 +344,9 @@ var resultTestCases = []resultTestCase{ expectedResult: &interpretedResult{ pairResults: map[DirectedNodePair]pairResult{ getTestPair(0, 1): successPairResult(100), + getTestPair(1, 2): successPairResult(99), }, - finalFailureReason: &reasonError, - nodeFailure: nil, + nodeFailure: nil, }, }, } From 35ebfcda9e9e31348c40bc41eae6575ea1da559d Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Mon, 6 Apr 2020 17:45:31 +0200 Subject: [PATCH 11/13] itest: create mpp test context --- lntest/itest/lnd_mpp_test.go | 245 ++++++++++++++++++++--------------- 1 file changed, 143 insertions(+), 102 deletions(-) diff --git a/lntest/itest/lnd_mpp_test.go b/lntest/itest/lnd_mpp_test.go index 9b338baf0..0b015b591 100644 --- a/lntest/itest/lnd_mpp_test.go +++ b/lntest/itest/lnd_mpp_test.go @@ -22,6 +22,9 @@ import ( func testSendToRouteMultiPath(net *lntest.NetworkHarness, t *harnessTest) { ctxb := context.Background() + ctx := newMppTestContext(t, net) + defer ctx.shutdownNodes() + // To ensure the payment goes through seperate paths, we'll set a // channel size that can only carry one shard at a time. We'll divide // the payment into 3 shards. @@ -38,113 +41,19 @@ func testSendToRouteMultiPath(net *lntest.NetworkHarness, t *harnessTest) { // \ / // \__ Dave ____/ // - // - // Create the three nodes in addition to Alice and Bob. - alice := net.Alice - bob := net.Bob - carol, err := net.NewNode("carol", nil) - if err != nil { - t.Fatalf("unable to create carol: %v", err) - } - defer shutdownAndAssert(net, t, carol) - - dave, err := net.NewNode("dave", nil) - if err != nil { - t.Fatalf("unable to create dave: %v", err) - } - defer shutdownAndAssert(net, t, dave) - - eve, err := net.NewNode("eve", nil) - if err != nil { - t.Fatalf("unable to create eve: %v", err) - } - defer shutdownAndAssert(net, t, eve) - - nodes := []*lntest.HarnessNode{alice, bob, carol, dave, eve} - - // Connect nodes to ensure propagation of channels. - for i := 0; i < len(nodes); i++ { - for j := i + 1; j < len(nodes); j++ { - ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) - if err := net.EnsureConnected(ctxt, nodes[i], nodes[j]); err != nil { - t.Fatalf("unable to connect nodes: %v", err) - } - } - } - - // We'll send shards along three routes from Alice. - sendRoutes := [][]*lntest.HarnessNode{ - {carol, bob}, - {dave, bob}, - {carol, eve, bob}, - } - - // Keep a list of all our active channels. - var networkChans []*lnrpc.ChannelPoint - var closeChannelFuncs []func() - - // openChannel is a helper to open a channel from->to. - openChannel := func(from, to *lntest.HarnessNode, chanSize btcutil.Amount) { - ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) - err := net.SendCoins(ctxt, btcutil.SatoshiPerBitcoin, from) - if err != nil { - t.Fatalf("unable to send coins : %v", err) - } - - ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) - chanPoint := openChannelAndAssert( - ctxt, t, net, from, to, - lntest.OpenChannelParams{ - Amt: chanSize, - }, - ) - - closeChannelFuncs = append(closeChannelFuncs, func() { - ctxt, _ := context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert( - ctxt, t, net, from, chanPoint, false, - ) - }) - - networkChans = append(networkChans, chanPoint) - } - - // Open channels between the nodes. - openChannel(carol, bob, chanAmt) - openChannel(dave, bob, chanAmt) - openChannel(alice, dave, chanAmt) - openChannel(eve, bob, chanAmt) - openChannel(carol, eve, chanAmt) + ctx.openChannel(ctx.carol, ctx.bob, chanAmt) + ctx.openChannel(ctx.dave, ctx.bob, chanAmt) + ctx.openChannel(ctx.alice, ctx.dave, chanAmt) + ctx.openChannel(ctx.eve, ctx.bob, chanAmt) + ctx.openChannel(ctx.carol, ctx.eve, chanAmt) // Since the channel Alice-> Carol will have to carry two // shards, we make it larger. - openChannel(alice, carol, chanAmt+shardAmt) + ctx.openChannel(ctx.alice, ctx.carol, chanAmt+shardAmt) - for _, f := range closeChannelFuncs { - defer f() - } + defer ctx.closeChannels() - // Wait for all nodes to have seen all channels. - for _, chanPoint := range networkChans { - for _, node := range nodes { - txid, err := lnd.GetChanPointFundingTxid(chanPoint) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - point := wire.OutPoint{ - Hash: *txid, - Index: chanPoint.OutputIndex, - } - - ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) - err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) - if err != nil { - t.Fatalf("(%d): timeout waiting for "+ - "channel(%s) open: %v", - node.NodeID, point, err) - } - } - } + ctx.waitForChannels() // Make Bob create an invoice for Alice to pay. payReqs, rHashes, invoices, err := createPayReqs( @@ -197,6 +106,13 @@ func testSendToRouteMultiPath(net *lntest.NetworkHarness, t *harnessTest) { return routeResp.Route, nil } + // We'll send shards along three routes from Alice. + sendRoutes := [][]*lntest.HarnessNode{ + {ctx.carol, ctx.bob}, + {ctx.dave, ctx.bob}, + {ctx.carol, ctx.eve, ctx.bob}, + } + responses := make(chan *routerrpc.SendToRouteResponse, len(sendRoutes)) for _, hops := range sendRoutes { // Build a route for the specified hops. @@ -355,3 +271,128 @@ func testSendToRouteMultiPath(net *lntest.NetworkHarness, t *harnessTest) { // ...and in Bob's list of paid invoices. assertSettledInvoice(net.Bob, rHash, 3) } + +type mppTestContext struct { + t *harnessTest + net *lntest.NetworkHarness + + // Keep a list of all our active channels. + networkChans []*lnrpc.ChannelPoint + closeChannelFuncs []func() + + alice, bob, carol, dave, eve *lntest.HarnessNode + nodes []*lntest.HarnessNode +} + +func newMppTestContext(t *harnessTest, + net *lntest.NetworkHarness) *mppTestContext { + + ctxb := context.Background() + + // Create a five-node context consisting of Alice, Bob and three new + // nodes. + carol, err := net.NewNode("carol", nil) + if err != nil { + t.Fatalf("unable to create carol: %v", err) + } + + dave, err := net.NewNode("dave", nil) + if err != nil { + t.Fatalf("unable to create dave: %v", err) + } + + eve, err := net.NewNode("eve", nil) + if err != nil { + t.Fatalf("unable to create eve: %v", err) + } + + // Connect nodes to ensure propagation of channels. + nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave, eve} + for i := 0; i < len(nodes); i++ { + for j := i + 1; j < len(nodes); j++ { + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + if err := net.EnsureConnected(ctxt, nodes[i], nodes[j]); err != nil { + t.Fatalf("unable to connect nodes: %v", err) + } + } + } + + ctx := mppTestContext{ + t: t, + net: net, + alice: net.Alice, + bob: net.Bob, + carol: carol, + dave: dave, + eve: eve, + nodes: nodes, + } + + return &ctx +} + +// openChannel is a helper to open a channel from->to. +func (c *mppTestContext) openChannel(from, to *lntest.HarnessNode, chanSize btcutil.Amount) { + ctxb := context.Background() + + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + err := c.net.SendCoins(ctxt, btcutil.SatoshiPerBitcoin, from) + if err != nil { + c.t.Fatalf("unable to send coins : %v", err) + } + + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPoint := openChannelAndAssert( + ctxt, c.t, c.net, from, to, + lntest.OpenChannelParams{ + Amt: chanSize, + }, + ) + + c.closeChannelFuncs = append(c.closeChannelFuncs, func() { + ctxt, _ := context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert( + ctxt, c.t, c.net, from, chanPoint, false, + ) + }) + + c.networkChans = append(c.networkChans, chanPoint) +} + +func (c *mppTestContext) closeChannels() { + for _, f := range c.closeChannelFuncs { + f() + } +} + +func (c *mppTestContext) shutdownNodes() { + shutdownAndAssert(c.net, c.t, c.carol) + shutdownAndAssert(c.net, c.t, c.dave) + shutdownAndAssert(c.net, c.t, c.eve) +} + +func (c *mppTestContext) waitForChannels() { + ctxb := context.Background() + + // Wait for all nodes to have seen all channels. + for _, chanPoint := range c.networkChans { + for _, node := range c.nodes { + txid, err := lnd.GetChanPointFundingTxid(chanPoint) + if err != nil { + c.t.Fatalf("unable to get txid: %v", err) + } + point := wire.OutPoint{ + Hash: *txid, + Index: chanPoint.OutputIndex, + } + + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) + if err != nil { + c.t.Fatalf("(%d): timeout waiting for "+ + "channel(%s) open: %v", + node.NodeID, point, err) + } + } + } +} From af14f2e425c4a6f8820d98bb5c41637cb1ebaf22 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Thu, 26 Mar 2020 15:05:11 +0100 Subject: [PATCH 12/13] itest: add multi-part payment test Co-authored-by: Johan T. Halseth --- lntest/itest/lnd_send_multi_path_payment.go | 141 ++++++++++++++++++++ lntest/itest/lnd_test.go | 46 +++++++ 2 files changed, 187 insertions(+) create mode 100644 lntest/itest/lnd_send_multi_path_payment.go diff --git a/lntest/itest/lnd_send_multi_path_payment.go b/lntest/itest/lnd_send_multi_path_payment.go new file mode 100644 index 000000000..b70564f07 --- /dev/null +++ b/lntest/itest/lnd_send_multi_path_payment.go @@ -0,0 +1,141 @@ +// +build rpctest + +package itest + +import ( + "context" + "encoding/hex" + + "github.com/btcsuite/btcutil" + "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" + "github.com/lightningnetwork/lnd/lntest" +) + +// testSendMultiPathPayment tests that we are able to successfully route a +// payment using multiple shards across different paths. +func testSendMultiPathPayment(net *lntest.NetworkHarness, t *harnessTest) { + ctxb := context.Background() + + ctx := newMppTestContext(t, net) + defer ctx.shutdownNodes() + + const paymentAmt = btcutil.Amount(300000) + + // Set up a network with three different paths Alice <-> Bob. Channel + // capacities are set such that the payment can only succeed if (at + // least) three paths are used. + // + // _ Eve _ + // / \ + // Alice -- Carol ---- Bob + // \ / + // \__ Dave ____/ + // + ctx.openChannel(ctx.carol, ctx.bob, 135000) + ctx.openChannel(ctx.alice, ctx.carol, 235000) + ctx.openChannel(ctx.dave, ctx.bob, 135000) + ctx.openChannel(ctx.alice, ctx.dave, 135000) + ctx.openChannel(ctx.eve, ctx.bob, 135000) + ctx.openChannel(ctx.carol, ctx.eve, 135000) + + defer ctx.closeChannels() + + ctx.waitForChannels() + + // Increase Dave's fee to make the test deterministic. Otherwise it + // would be unpredictable whether pathfinding would go through Charlie + // or Dave for the first shard. + _, err := ctx.dave.UpdateChannelPolicy( + context.Background(), + &lnrpc.PolicyUpdateRequest{ + Scope: &lnrpc.PolicyUpdateRequest_Global{Global: true}, + BaseFeeMsat: 500000, + FeeRate: 0.001, + TimeLockDelta: 40, + }, + ) + if err != nil { + t.Fatalf("dave policy update: %v", err) + } + // Our first test will be Alice paying Bob using a SendPayment call. + // Let Bob create an invoice for Alice to pay. + payReqs, rHashes, invoices, err := createPayReqs( + net.Bob, paymentAmt, 1, + ) + if err != nil { + t.Fatalf("unable to create pay reqs: %v", err) + } + + rHash := rHashes[0] + payReq := payReqs[0] + + payment := sendAndAssertSuccess( + t, net.Alice, + &routerrpc.SendPaymentRequest{ + PaymentRequest: payReq, + MaxHtlcs: 10, + TimeoutSeconds: 60, + FeeLimitMsat: noFeeLimitMsat, + }, + ) + + // Make sure we got the preimage. + if payment.PaymentPreimage != hex.EncodeToString(invoices[0].RPreimage) { + t.Fatalf("preimage doesn't match") + } + + // Check that Alice split the payment in at least three shards. Because + // the hand-off of the htlc to the link is asynchronous (via a mailbox), + // there is some non-determinism in the process. Depending on whether + // the new pathfinding round is started before or after the htlc is + // locked into the channel, different sharding may occur. Therefore we + // can only check if the number of shards isn't below the theoretical + // minimum. + succeeded := 0 + for _, htlc := range payment.Htlcs { + if htlc.Status == lnrpc.HTLCAttempt_SUCCEEDED { + succeeded++ + } + } + + const minExpectedShards = 3 + if succeeded < minExpectedShards { + t.Fatalf("expected at least %v shards, but got %v", + minExpectedShards, succeeded) + } + + // Make sure Bob show the invoice as settled for the full + // amount. + ctxt, _ := context.WithTimeout(ctxb, defaultTimeout) + inv, err := ctx.bob.LookupInvoice( + ctxt, &lnrpc.PaymentHash{ + RHash: rHash, + }, + ) + if err != nil { + t.Fatalf("error when obtaining invoice: %v", err) + } + + if inv.AmtPaidSat != int64(paymentAmt) { + t.Fatalf("incorrect payment amt for invoice"+ + "want: %d, got %d", + paymentAmt, inv.AmtPaidSat) + } + + if inv.State != lnrpc.Invoice_SETTLED { + t.Fatalf("Invoice not settled: %v", inv.State) + } + + settled := 0 + for _, htlc := range inv.Htlcs { + if htlc.State == lnrpc.InvoiceHTLCState_SETTLED { + settled++ + } + + } + if settled != succeeded { + t.Fatalf("expected invoice to be settled "+ + "with %v HTLCs, had %v", succeeded, settled) + } +} diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go index 4e2be7257..d3365ab3b 100644 --- a/lntest/itest/lnd_test.go +++ b/lntest/itest/lnd_test.go @@ -61,6 +61,7 @@ const ( channelCloseTimeout = lntest.ChannelCloseTimeout itestLndBinary = "../../lnd-itest" anchorSize = 330 + noFeeLimitMsat = math.MaxInt64 ) // harnessTest wraps a regular testing.T providing enhanced error detection @@ -14718,6 +14719,47 @@ func testExternalFundingChanPoint(net *lntest.NetworkHarness, t *harnessTest) { closeChannelAndAssert(ctxt, t, net, dave, chanPoint, false) } +// sendAndAssertSuccess sends the given payment requests and asserts that the +// payment completes successfully. +func sendAndAssertSuccess(t *harnessTest, node *lntest.HarnessNode, + req *routerrpc.SendPaymentRequest) *lnrpc.Payment { + + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + + stream, err := node.RouterClient.SendPayment(ctx, req) + if err != nil { + t.Fatalf("unable to send payment: %v", err) + } + + result, err := getPaymentResult(stream) + if err != nil { + t.Fatalf("unable to get payment result: %v", err) + } + + if result.Status != lnrpc.Payment_SUCCEEDED { + t.Fatalf("payment failed: %v", result.Status) + } + + return result +} + +// getPaymentResult reads a final result from the stream and returns it. +func getPaymentResult(stream routerrpc.Router_SendPaymentClient) ( + *lnrpc.Payment, error) { + + for { + payment, err := stream.Recv() + if err != nil { + return nil, err + } + + if payment.Status != lnrpc.Payment_IN_FLIGHT { + return payment, nil + } + } +} + type testCase struct { name string test func(net *lntest.NetworkHarness, t *harnessTest) @@ -14959,6 +15001,10 @@ var testsCases = []*testCase{ name: "sendtoroute multi path payment", test: testSendToRouteMultiPath, }, + { + name: "send multi path payment", + test: testSendMultiPathPayment, + }, } // TestLightningNetworkDaemon performs a series of integration tests amongst a From ef38f8f2c74019cbfba4ed2a1a3b781e98f7e1f9 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Mon, 6 Apr 2020 21:32:26 +0200 Subject: [PATCH 13/13] itest: log failure time --- lntest/itest/lnd_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go index d3365ab3b..5f0c958db 100644 --- a/lntest/itest/lnd_test.go +++ b/lntest/itest/lnd_test.go @@ -15145,6 +15145,10 @@ func TestLightningNetworkDaemon(t *testing.T) { // Stop at the first failure. Mimic behavior of original test // framework. if !success { + // Log failure time to help relate the lnd logs to the + // failure. + t.Logf("Failure time: %v", + time.Now().Format("2006-01-02 15:04:05.000")) break } }