routerrpc+routing: adapt payment session for multi shard send

Modifies the payment session to launch additional pathfinding attempts
for lower amounts. If a single shot payment isn't possible, the goal is
to try to complete the payment using multiple htlcs. In previous
commits, the payment lifecycle has been prepared to deal with
partial-amount routes returned from the payment session. It will query
for additional shards if needed.

Additionally a new rpc payment parameter is added that controls the
maximum number of shards that will be used for the payment.

routing/integrated_routing_context_test.go

@@ -1,7 +1,9 @@
 package routing
 
 import (
+	"fmt"
 	"io/ioutil"
+	"math"
 	"os"
 	"testing"
 	"time"
@@ -64,6 +66,7 @@ func newIntegratedRoutingContext(t *testing.T) *integratedRoutingContext {
 
 		pathFindingCfg: PathFindingConfig{
 			PaymentAttemptPenalty: 1000,
+			MinProbability:        0.01,
 		},
 
 		source: source,
@@ -79,9 +82,15 @@ type htlcAttempt struct {
 	success bool
 }
 
+func (h htlcAttempt) String() string {
+	return fmt.Sprintf("success=%v, route=%v", h.success, h.route)
+}
+
 // testPayment launches a test payment and asserts that it is completed after
 // the expected number of attempts.
-func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) {
+func (c *integratedRoutingContext) testPayment(maxHtlcs uint32) ([]htlcAttempt,
+	error) {
+
 	var (
 		nextPid  uint64
 		attempts []htlcAttempt
@@ -119,10 +128,16 @@ func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) {
 		return bandwidthHints, nil
 	}
 
+	var paymentAddr [32]byte
 	payment := LightningPayment{
 		FinalCLTVDelta: uint16(c.finalExpiry),
 		FeeLimit:       lnwire.MaxMilliSatoshi,
 		Target:         c.target.pubkey,
+		PaymentAddr:    &paymentAddr,
+		DestFeatures:   lnwire.NewFeatureVector(mppFeatures, nil),
+		Amount:         c.amt,
+		CltvLimit:      math.MaxUint32,
+		MaxHtlcs:       maxHtlcs,
 	}
 
 	session := &paymentSession{
@@ -134,10 +149,15 @@ func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) {
 		},
 		pathFindingConfig: c.pathFindingCfg,
 		missionControl:    mc,
+		minShardAmt:       lnwire.NewMSatFromSatoshis(5000),
 	}
 
 	// Now the payment control loop starts. It will keep trying routes until
 	// the payment succeeds.
+	var (
+		amtRemaining  = payment.Amount
+		inFlightHtlcs uint32
+	)
 	for {
 		// Create bandwidth hints based on local channel balances.
 		bandwidthHints := map[uint64]lnwire.MilliSatoshi{}
@@ -147,10 +167,10 @@ func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) {
 
 		// Find a route.
 		route, err := session.RequestRoute(
-			c.amt, lnwire.MaxMilliSatoshi, 0, 0,
+			amtRemaining, lnwire.MaxMilliSatoshi, inFlightHtlcs, 0,
 		)
 		if err != nil {
-			return nil, err
+			return attempts, err
 		}
 
 		// Send out the htlc on the mock graph.
@@ -167,21 +187,33 @@ func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) {
 			success: success,
 		})
 
-		// Process the result.
+		// Process the result. In normal Lightning operations, the
+		// sender doesn't get an acknowledgement from the recipient that
+		// the htlc arrived. In integrated routing tests, this
+		// acknowledgement is available. It is a simplification of
+		// reality that still allows certain classes of tests to be
+		// performed.
 		if success {
+			inFlightHtlcs++
+
 			err := mc.ReportPaymentSuccess(pid, route)
 			if err != nil {
 				c.t.Fatal(err)
 			}
 
-			// If the payment is successful, the control loop can be
-			// broken out of.
-			break
+			amtRemaining -= route.ReceiverAmt()
+
+			// If the full amount has been paid, the payment is
+			// successful and the control loop can be terminated.
+			if amtRemaining == 0 {
+				break
+			}
+
+			// Otherwise try to send the remaining amount.
+			continue
 		}
 
 		// Failure, update mission control and retry.
-		c.t.Logf("fail: %v @ %v\n", htlcResult.failure, htlcResult.failureSource)
-
 		finalResult, err := mc.ReportPaymentFail(
 			pid, route,
 			getNodeIndex(route, htlcResult.failureSource),
@@ -192,13 +224,10 @@ func (c *integratedRoutingContext) testPayment() ([]htlcAttempt, error) {
 		}
 
 		if finalResult != nil {
-			c.t.Logf("final result: %v\n", finalResult)
 			break
 		}
 	}
 
-	c.t.Logf("Payment attempts: %v\n", len(attempts))
-
 	return attempts, nil
 }
 

routing/integrated_routing_test.go

@@ -2,6 +2,9 @@ package routing
 
 import (
 	"testing"
+
+	"github.com/btcsuite/btcutil"
+	"github.com/lightningnetwork/lnd/lnwire"
 )
 
 // TestProbabilityExtrapolation tests that probabilities for tried channels are
@@ -50,7 +53,7 @@ func TestProbabilityExtrapolation(t *testing.T) {
 	// a specific number of attempts to safe-guard against accidental
 	// modifications anywhere in the chain of components that is involved in
 	// this test.
-	attempts, err := ctx.testPayment()
+	attempts, err := ctx.testPayment(1)
 	if err != nil {
 		t.Fatalf("payment failed: %v", err)
 	}
@@ -62,7 +65,7 @@ func TestProbabilityExtrapolation(t *testing.T) {
 	// of data from other channels), all ten bad channels will be tried
 	// first before switching to the paid channel.
 	ctx.mcCfg.AprioriWeight = 1
-	attempts, err = ctx.testPayment()
+	attempts, err = ctx.testPayment(1)
 	if err != nil {
 		t.Fatalf("payment failed: %v", err)
 	}
@@ -70,3 +73,211 @@ func TestProbabilityExtrapolation(t *testing.T) {
 		t.Fatalf("expected 11 attempts, but needed %v", len(attempts))
 	}
 }
+
+type mppSendTestCase struct {
+	name             string
+	amt              btcutil.Amount
+	expectedAttempts int
+
+	// expectedSuccesses is a list of htlcs that made it to the receiver,
+	// regardless of whether the final set became complete or not.
+	expectedSuccesses []expectedHtlcSuccess
+
+	graph           func(g *mockGraph)
+	expectedFailure bool
+	maxHtlcs        uint32
+}
+
+const (
+	chanSourceIm1 = 13
+	chanIm1Target = 32
+	chanSourceIm2 = 14
+	chanIm2Target = 42
+)
+
+func onePathGraph(g *mockGraph) {
+	// Create the following network of nodes:
+	// source -> intermediate1 -> target
+
+	const im1NodeID = 3
+	intermediate1 := newMockNode(im1NodeID)
+	g.addNode(intermediate1)
+
+	g.addChannel(chanSourceIm1, sourceNodeID, im1NodeID, 200000)
+	g.addChannel(chanIm1Target, targetNodeID, im1NodeID, 100000)
+}
+
+func twoPathGraph(g *mockGraph) {
+	// Create the following network of nodes:
+	// source -> intermediate1 -> target
+	// source -> intermediate2 -> target
+
+	const im1NodeID = 3
+	intermediate1 := newMockNode(im1NodeID)
+	g.addNode(intermediate1)
+
+	const im2NodeID = 4
+	intermediate2 := newMockNode(im2NodeID)
+	g.addNode(intermediate2)
+
+	g.addChannel(chanSourceIm1, sourceNodeID, im1NodeID, 200000)
+	g.addChannel(chanSourceIm2, sourceNodeID, im2NodeID, 200000)
+	g.addChannel(chanIm1Target, targetNodeID, im1NodeID, 100000)
+	g.addChannel(chanIm2Target, targetNodeID, im2NodeID, 100000)
+}
+
+var mppTestCases = []mppSendTestCase{
+	// Test a two-path graph with sufficient liquidity. It is expected that
+	// pathfinding will try first try to send the full amount via the two
+	// available routes. When that fails, it will half the amount to 35k sat
+	// and retry. That attempt reaches the target successfully. Then the
+	// same route is tried again. Because the channel only had 50k sat, it
+	// will fail. Finally the second route is tried for 35k and it succeeds
+	// too. Mpp payment complete.
+	{
+
+		name:             "sufficient inbound",
+		graph:            twoPathGraph,
+		amt:              70000,
+		expectedAttempts: 5,
+		expectedSuccesses: []expectedHtlcSuccess{
+			{
+				amt:   35000,
+				chans: []uint64{chanSourceIm1, chanIm1Target},
+			},
+			{
+				amt:   35000,
+				chans: []uint64{chanSourceIm2, chanIm2Target},
+			},
+		},
+		maxHtlcs: 1000,
+	},
+
+	// Test that a cap on the max htlcs makes it impossible to pay.
+	{
+		name:              "no splitting",
+		graph:             twoPathGraph,
+		amt:               70000,
+		expectedAttempts:  2,
+		expectedSuccesses: []expectedHtlcSuccess{},
+		expectedFailure:   true,
+		maxHtlcs:          1,
+	},
+
+	// Test that an attempt is made to split the payment in multiple parts
+	// that all use the same route if the full amount cannot be sent in a
+	// single htlc. The sender is effectively probing the receiver's
+	// incoming channel to see if it has sufficient balance. In this test
+	// case, the endeavour fails.
+	{
+
+		name:             "one path split",
+		graph:            onePathGraph,
+		amt:              70000,
+		expectedAttempts: 7,
+		expectedSuccesses: []expectedHtlcSuccess{
+			{
+				amt:   35000,
+				chans: []uint64{chanSourceIm1, chanIm1Target},
+			},
+			{
+				amt:   8750,
+				chans: []uint64{chanSourceIm1, chanIm1Target},
+			},
+		},
+		expectedFailure: true,
+		maxHtlcs:        1000,
+	},
+}
+
+// TestMppSend tests that a payment can be completed using multiple shards.
+func TestMppSend(t *testing.T) {
+	for _, testCase := range mppTestCases {
+		testCase := testCase
+
+		t.Run(testCase.name, func(t *testing.T) {
+			testMppSend(t, &testCase)
+		})
+	}
+}
+
+func testMppSend(t *testing.T, testCase *mppSendTestCase) {
+	ctx := newIntegratedRoutingContext(t)
+
+	g := ctx.graph
+	testCase.graph(g)
+
+	ctx.amt = lnwire.NewMSatFromSatoshis(testCase.amt)
+
+	attempts, err := ctx.testPayment(testCase.maxHtlcs)
+	switch {
+	case err == nil && testCase.expectedFailure:
+		t.Fatal("expected payment to fail")
+	case err != nil && !testCase.expectedFailure:
+		t.Fatal("expected payment to succeed")
+	}
+
+	if len(attempts) != testCase.expectedAttempts {
+		t.Fatalf("expected %v attempts, but needed %v",
+			testCase.expectedAttempts, len(attempts),
+		)
+	}
+
+	assertSuccessAttempts(t, attempts, testCase.expectedSuccesses)
+}
+
+// expectedHtlcSuccess describes an expected successful htlc attempt.
+type expectedHtlcSuccess struct {
+	amt   btcutil.Amount
+	chans []uint64
+}
+
+// equals matches the expectation with an actual attempt.
+func (e *expectedHtlcSuccess) equals(a htlcAttempt) bool {
+	if a.route.TotalAmount !=
+		lnwire.NewMSatFromSatoshis(e.amt) {
+
+		return false
+	}
+
+	if len(a.route.Hops) != len(e.chans) {
+		return false
+	}
+
+	for i, h := range a.route.Hops {
+		if h.ChannelID != e.chans[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+// assertSuccessAttempts asserts that the set of successful htlc attempts
+// matches the given expectation.
+func assertSuccessAttempts(t *testing.T, attempts []htlcAttempt,
+	expected []expectedHtlcSuccess) {
+
+	successCount := 0
+loop:
+	for _, a := range attempts {
+		if !a.success {
+			continue
+		}
+
+		successCount++
+
+		for _, exp := range expected {
+			if exp.equals(a) {
+				continue loop
+			}
+		}
+
+		t.Fatalf("htlc success %v not found", a)
+	}
+
+	if successCount != len(expected) {
+		t.Fatalf("expected %v successful htlcs, but got %v",
+			expected, successCount)
+	}
+}

routing/pathfind_test.go

@@ -81,6 +81,12 @@ var (
 		), lnwire.Features,
 	)
 
+	mppFeatures = lnwire.NewRawFeatureVector(
+		lnwire.TLVOnionPayloadOptional,
+		lnwire.PaymentAddrOptional,
+		lnwire.MPPOptional,
+	)
+
 	unknownRequiredFeatures = lnwire.NewFeatureVector(
 		lnwire.NewRawFeatureVector(100), lnwire.Features,
 	)

routing/payment_session.go

@@ -31,6 +31,13 @@ const (
 	errEmptyPaySession
 )
 
+var (
+	// DefaultShardMinAmt is the default amount beyond which we won't try to
+	// further split the payment if no route is found. It is the minimum
+	// amount that we use as the shard size when splitting.
+	DefaultShardMinAmt = lnwire.NewMSatFromSatoshis(10000)
+)
+
 // Error returns the string representation of the noRouteError
 func (e noRouteError) Error() string {
 	switch e {
@@ -111,6 +118,12 @@ type paymentSession struct {
 	pathFindingConfig PathFindingConfig
 
 	missionControl MissionController
+
+	// minShardAmt is the amount beyond which we won't try to further split
+	// the payment if no route is found. If the maximum number of htlcs
+	// specified in the payment is one, under no circumstances splitting
+	// will happen and this value remains unused.
+	minShardAmt lnwire.MilliSatoshi
 }
 
 // RequestRoute returns a route which is likely to be capable for successfully
@@ -155,58 +168,87 @@ func (p *paymentSession) RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi,
 		PaymentAddr:       p.payment.PaymentAddr,
 	}
 
-	// We'll also obtain a set of bandwidthHints from the lower layer for
-	// each of our outbound channels. This will allow the path finding to
-	// skip any links that aren't active or just don't have enough bandwidth
-	// to carry the payment. New bandwidth hints are queried for every new
-	// path finding attempt, because concurrent payments may change
-	// balances.
-	bandwidthHints, err := p.getBandwidthHints()
-	if err != nil {
-		return nil, err
-	}
-
 	finalHtlcExpiry := int32(height) + int32(finalCltvDelta)
 
-	routingGraph, cleanup, err := p.getRoutingGraph()
-	if err != nil {
-		return nil, err
+	for {
+		// We'll also obtain a set of bandwidthHints from the lower
+		// layer for each of our outbound channels. This will allow the
+		// path finding to skip any links that aren't active or just
+		// don't have enough bandwidth to carry the payment. New
+		// bandwidth hints are queried for every new path finding
+		// attempt, because concurrent payments may change balances.
+		bandwidthHints, err := p.getBandwidthHints()
+		if err != nil {
+			return nil, err
+		}
+
+		log.Debugf("PaymentSession for %x: trying pathfinding with %v",
+			p.payment.PaymentHash, maxAmt)
+
+		// Get a routing graph.
+		routingGraph, cleanup, err := p.getRoutingGraph()
+		if err != nil {
+			return nil, err
+		}
+
+		sourceVertex := routingGraph.sourceNode()
+
+		// Find a route for the current amount.
+		path, err := p.pathFinder(
+			&graphParams{
+				additionalEdges: p.additionalEdges,
+				bandwidthHints:  bandwidthHints,
+				graph:           routingGraph,
+			},
+			restrictions, &p.pathFindingConfig,
+			sourceVertex, p.payment.Target,
+			maxAmt, finalHtlcExpiry,
+		)
+
+		// Close routing graph.
+		cleanup()
+
+		switch {
+		case err == errNoPathFound:
+			// No splitting if this is the last shard.
+			isLastShard := activeShards+1 >= p.payment.MaxHtlcs
+			if isLastShard {
+				return nil, errNoPathFound
+			}
+
+			// This is where the magic happens. If we can't find a
+			// route, try it for half the amount.
+			maxAmt /= 2
+
+			// Put a lower bound on the minimum shard size.
+			if maxAmt < p.minShardAmt {
+				return nil, errNoPathFound
+			}
+
+			// Go pathfinding.
+			continue
+
+		case err != nil:
+			return nil, err
+		}
+
+		// With the next candidate path found, we'll attempt to turn
+		// this into a route by applying the time-lock and fee
+		// requirements.
+		route, err := newRoute(
+			sourceVertex, path, height,
+			finalHopParams{
+				amt:         maxAmt,
+				totalAmt:    p.payment.Amount,
+				cltvDelta:   finalCltvDelta,
+				records:     p.payment.DestCustomRecords,
+				paymentAddr: p.payment.PaymentAddr,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		return route, err
 	}
-	defer cleanup()
-
-	sourceVertex := routingGraph.sourceNode()
-
-	path, err := p.pathFinder(
-		&graphParams{
-			additionalEdges: p.additionalEdges,
-			bandwidthHints:  bandwidthHints,
-			graph:           routingGraph,
-		},
-		restrictions, &p.pathFindingConfig,
-		sourceVertex, p.payment.Target,
-		maxAmt, finalHtlcExpiry,
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	// With the next candidate path found, we'll attempt to turn this into
-	// a route by applying the time-lock and fee requirements.
-	route, err := newRoute(
-		sourceVertex, path, height,
-		finalHopParams{
-			amt:         maxAmt,
-			totalAmt:    maxAmt,
-			cltvDelta:   finalCltvDelta,
-			records:     p.payment.DestCustomRecords,
-			paymentAddr: p.payment.PaymentAddr,
-		},
-	)
-	if err != nil {
-		// TODO(roasbeef): return which edge/vertex didn't work
-		// out
-		return nil, err
-	}
-
-	return route, err
 }

routing/payment_session_source.go

@@ -86,6 +86,7 @@ func (m *SessionSource) NewPaymentSession(p *LightningPayment) (
 		getRoutingGraph:   m.getRoutingGraph,
 		pathFindingConfig: m.PathFindingConfig,
 		missionControl:    m.MissionControl,
+		minShardAmt:       DefaultShardMinAmt,
 	}, nil
 }
 

routing/router.go

@@ -1624,6 +1624,10 @@ type LightningPayment struct {
 	// understand this new onion payload format, then the payment will
 	// fail.
 	DestCustomRecords record.CustomSet
+
+	// MaxHtlcs is the maximum number of partial payments that may be use to
+	// complete the full amount.
+	MaxHtlcs uint32
 }
 
 // SendPayment attempts to send a payment as described within the passed