Merge pull request #2802 from joostjager/probability

routing: probability based path finding

routing/heap.go

@@ -25,8 +25,14 @@ type nodeWithDist struct {
 	// node. This value does not include the final cltv.
 	incomingCltv uint32
 
-	// fee is the fee that this node is charging for forwarding.
-	fee lnwire.MilliSatoshi
+	// probability is the probability that from this node onward the route
+	// is successful.
+	probability float64
+
+	// weight is the cost of the route from this node to the destination.
+	// Includes the routing fees and a virtual cost factor to account for
+	// time locks.
+	weight int64
 }
 
 // distanceHeap is a min-distance heap that's used within our path finding

routing/missioncontrol.go

@@ -1,6 +1,7 @@
 package routing
 
 import (
+	"math"
 	"sync"
 	"time"
 
@@ -13,48 +14,23 @@ import (
 )
 
 const (
-	// vertexDecay is the decay period of colored vertexes added to
-	// MissionControl. Once vertexDecay passes after an entry has been
-	// added to the prune view, it is garbage collected. This value is
-	// larger than edgeDecay as an edge failure typical indicates an
-	// unbalanced channel, while a vertex failure indicates a node is not
-	// online and active.
-	vertexDecay = time.Duration(time.Minute * 5)
-
-	// edgeDecay is the decay period of colored edges added to
-	// MissionControl. Once edgeDecay passed after an entry has been added,
-	// it is garbage collected. This value is smaller than vertexDecay as
-	// an edge related failure during payment sending typically indicates
-	// that a channel was unbalanced, a condition which may quickly change.
-	//
-	// TODO(roasbeef): instead use random delay on each?
-	edgeDecay = time.Duration(time.Second * 5)
+	// DefaultPenaltyHalfLife is the default half-life duration. The
+	// half-life duration defines after how much time a penalized node or
+	// channel is back at 50% probability.
+	DefaultPenaltyHalfLife = time.Hour
 )
 
 // MissionControl contains state which summarizes the past attempts of HTLC
-// routing by external callers when sending payments throughout the network.
-// MissionControl remembers the outcome of these past routing attempts (success
-// and failure), and is able to provide hints/guidance to future HTLC routing
-// attempts.  MissionControl maintains a decaying network view of the
-// edges/vertexes that should be marked as "pruned" during path finding. This
-// graph view acts as a shared memory during HTLC payment routing attempts.
-// With each execution, if an error is encountered, based on the type of error
-// and the location of the error within the route, an edge or vertex is added
-// to the view. Later sending attempts will then query the view for all the
-// vertexes/edges that should be ignored. Items in the view decay after a set
-// period of time, allowing the view to be dynamic w.r.t network changes.
+// routing by external callers when sending payments throughout the network. It
+// acts as a shared memory during routing attempts with the goal to optimize the
+// payment attempt success rate.
+//
+// Failed payment attempts are reported to mission control. These reports are
+// used to track the time of the last node or channel level failure. The time
+// since the last failure is used to estimate a success probability that is fed
+// into the path finding process for subsequent payment attempts.
 type MissionControl struct {
-	// failedEdges maps a short channel ID to be pruned, to the time that
-	// it was added to the prune view. Edges are added to this map if a
-	// caller reports to MissionControl a failure localized to that edge
-	// when sending a payment.
-	failedEdges map[EdgeLocator]time.Time
-
-	// failedVertexes maps a node's public key that should be pruned, to
-	// the time that it was added to the prune view. Vertexes are added to
-	// this map if a caller reports to MissionControl a failure localized
-	// to that particular vertex.
-	failedVertexes map[route.Vertex]time.Time
+	history map[route.Vertex]*nodeHistory
 
 	graph *channeldb.ChannelGraph
 
@@ -62,6 +38,12 @@ type MissionControl struct {
 
 	queryBandwidth func(*channeldb.ChannelEdgeInfo) lnwire.MilliSatoshi
 
+	// now is expected to return the current time. It is supplied as an
+	// external function to enable deterministic unit tests.
+	now func() time.Time
+
+	cfg *MissionControlConfig
+
 	sync.Mutex
 
 	// TODO(roasbeef): further counters, if vertex continually unavailable,
@@ -74,83 +56,113 @@ type MissionControl struct {
 // PaymentSessionSource interface.
 var _ PaymentSessionSource = (*MissionControl)(nil)
 
-// NewMissionControl returns a new instance of MissionControl.
+// MissionControlConfig defines parameters that control mission control
+// behaviour.
+type MissionControlConfig struct {
+	// PenaltyHalfLife defines after how much time a penalized node or
+	// channel is back at 50% probability.
+	PenaltyHalfLife time.Duration
+
+	// PaymentAttemptPenalty is the virtual cost in path finding weight
+	// units of executing a payment attempt that fails. It is used to trade
+	// off potentially better routes against their probability of
+	// succeeding.
+	PaymentAttemptPenalty lnwire.MilliSatoshi
+
+	// MinProbability defines the minimum success probability of the
+	// returned route.
+	MinRouteProbability float64
+
+	// AprioriHopProbability is the assumed success probability of a hop in
+	// a route when no other information is available.
+	AprioriHopProbability float64
+}
+
+// nodeHistory contains a summary of payment attempt outcomes involving a
+// particular node.
+type nodeHistory struct {
+	// lastFail is the last time a node level failure occurred, if any.
+	lastFail *time.Time
+
+	// channelLastFail tracks history per channel, if available for that
+	// channel.
+	channelLastFail map[uint64]*channelHistory
+}
+
+// channelHistory contains a summary of payment attempt outcomes involving a
+// particular channel.
+type channelHistory struct {
+	// lastFail is the last time a channel level failure occurred.
+	lastFail time.Time
+
+	// minPenalizeAmt is the minimum amount for which to take this failure
+	// into account.
+	minPenalizeAmt lnwire.MilliSatoshi
+}
+
+// MissionControlSnapshot contains a snapshot of the current state of mission
+// control.
+type MissionControlSnapshot struct {
+	// Nodes contains the per node information of this snapshot.
+	Nodes []MissionControlNodeSnapshot
+}
+
+// MissionControlNodeSnapshot contains a snapshot of the current node state in
+// mission control.
+type MissionControlNodeSnapshot struct {
+	// Node pubkey.
+	Node route.Vertex
+
+	// Lastfail is the time of last failure, if any.
+	LastFail *time.Time
+
+	// Channels is a list of channels for which specific information is
+	// logged.
+	Channels []MissionControlChannelSnapshot
+
+	// OtherChanSuccessProb is the success probability for channels not in
+	// the Channels slice.
+	OtherChanSuccessProb float64
+}
+
+// MissionControlChannelSnapshot contains a snapshot of the current channel
+// state in mission control.
+type MissionControlChannelSnapshot struct {
+	// ChannelID is the short channel id of the snapshot.
+	ChannelID uint64
+
+	// LastFail is the time of last failure.
+	LastFail time.Time
+
+	// MinPenalizeAmt is the minimum amount for which the channel will be
+	// penalized.
+	MinPenalizeAmt lnwire.MilliSatoshi
+
+	// SuccessProb is the success probability estimation for this channel.
+	SuccessProb float64
+}
+
+// NewMissionControl returns a new instance of missionControl.
 //
 // TODO(roasbeef): persist memory
 func NewMissionControl(g *channeldb.ChannelGraph, selfNode *channeldb.LightningNode,
-	qb func(*channeldb.ChannelEdgeInfo) lnwire.MilliSatoshi) *MissionControl {
+	qb func(*channeldb.ChannelEdgeInfo) lnwire.MilliSatoshi,
+	cfg *MissionControlConfig) *MissionControl {
+
+	log.Debugf("Instantiating mission control with config: "+
+		"PenaltyHalfLife=%v, PaymentAttemptPenalty=%v, "+
+		"MinRouteProbability=%v, AprioriHopProbability=%v",
+		cfg.PenaltyHalfLife,
+		int64(cfg.PaymentAttemptPenalty.ToSatoshis()),
+		cfg.MinRouteProbability, cfg.AprioriHopProbability)
 
 	return &MissionControl{
-		failedEdges:    make(map[EdgeLocator]time.Time),
-		failedVertexes: make(map[route.Vertex]time.Time),
+		history:        make(map[route.Vertex]*nodeHistory),
 		selfNode:       selfNode,
 		queryBandwidth: qb,
 		graph:          g,
-	}
-}
-
-// graphPruneView is a filter of sorts that path finding routines should
-// consult during the execution. Any edges or vertexes within the view should
-// be ignored during path finding. The contents of the view reflect the current
-// state of the wider network from the PoV of mission control compiled via HTLC
-// routing attempts in the past.
-type graphPruneView struct {
-	edges map[EdgeLocator]struct{}
-
-	vertexes map[route.Vertex]struct{}
-}
-
-// graphPruneView returns a new graphPruneView instance which is to be
-// consulted during path finding. If a vertex/edge is found within the returned
-// prune view, it is to be ignored as a goroutine has had issues routing
-// through it successfully. Within this method the main view of the
-// MissionControl is garbage collected as entries are detected to be "stale".
-func (m *MissionControl) graphPruneView() graphPruneView {
-	// First, we'll grab the current time, this value will be used to
-	// determine if an entry is stale or not.
-	now := time.Now()
-
-	m.Lock()
-
-	// For each of the vertexes that have been added to the prune view, if
-	// it is now "stale", then we'll ignore it and avoid adding it to the
-	// view we'll return.
-	vertexes := make(map[route.Vertex]struct{})
-	for vertex, pruneTime := range m.failedVertexes {
-		if now.Sub(pruneTime) >= vertexDecay {
-			log.Tracef("Pruning decayed failure report for vertex %v "+
-				"from Mission Control", vertex)
-
-			delete(m.failedVertexes, vertex)
-			continue
-		}
-
-		vertexes[vertex] = struct{}{}
-	}
-
-	// We'll also do the same for edges, but use the edgeDecay this time
-	// rather than the decay for vertexes.
-	edges := make(map[EdgeLocator]struct{})
-	for edge, pruneTime := range m.failedEdges {
-		if now.Sub(pruneTime) >= edgeDecay {
-			log.Tracef("Pruning decayed failure report for edge %v "+
-				"from Mission Control", edge)
-
-			delete(m.failedEdges, edge)
-			continue
-		}
-
-		edges[edge] = struct{}{}
-	}
-
-	m.Unlock()
-
-	log.Debugf("Mission Control returning prune view of %v edges, %v "+
-		"vertexes", len(edges), len(vertexes))
-
-	return graphPruneView{
-		edges:    edges,
-		vertexes: vertexes,
+		now:            time.Now,
+		cfg:            cfg,
 	}
 }
 
@@ -161,8 +173,6 @@ func (m *MissionControl) graphPruneView() graphPruneView {
 func (m *MissionControl) NewPaymentSession(routeHints [][]zpay32.HopHint,
 	target route.Vertex) (PaymentSession, error) {
 
-	viewSnapshot := m.graphPruneView()
-
 	edges := make(map[route.Vertex][]*channeldb.ChannelEdgePolicy)
 
 	// Traverse through all of the available hop hints and include them in
@@ -226,10 +236,9 @@ func (m *MissionControl) NewPaymentSession(routeHints [][]zpay32.HopHint,
 	}
 
 	return &paymentSession{
-		pruneViewSnapshot:    viewSnapshot,
 		additionalEdges:      edges,
 		bandwidthHints:       bandwidthHints,
-		errFailedPolicyChans: make(map[EdgeLocator]struct{}),
+		errFailedPolicyChans: make(map[nodeChannel]struct{}),
 		mc:                   m,
 		pathFinder:           findPath,
 	}, nil
@@ -239,8 +248,7 @@ func (m *MissionControl) NewPaymentSession(routeHints [][]zpay32.HopHint,
 // used for failure reporting to missioncontrol.
 func (m *MissionControl) NewPaymentSessionForRoute(preBuiltRoute *route.Route) PaymentSession {
 	return &paymentSession{
-		pruneViewSnapshot:    m.graphPruneView(),
-		errFailedPolicyChans: make(map[EdgeLocator]struct{}),
+		errFailedPolicyChans: make(map[nodeChannel]struct{}),
 		mc:                   m,
 		preBuiltRoute:        preBuiltRoute,
 	}
@@ -251,8 +259,7 @@ func (m *MissionControl) NewPaymentSessionForRoute(preBuiltRoute *route.Route) P
 // missioncontrol for resumed payment we don't want to make more attempts for.
 func (m *MissionControl) NewPaymentSessionEmpty() PaymentSession {
 	return &paymentSession{
-		pruneViewSnapshot:    m.graphPruneView(),
-		errFailedPolicyChans: make(map[EdgeLocator]struct{}),
+		errFailedPolicyChans: make(map[nodeChannel]struct{}),
 		mc:                   m,
 		preBuiltRoute:        &route.Route{},
 		preBuiltRouteTried:   true,
@@ -298,7 +305,174 @@ func generateBandwidthHints(sourceNode *channeldb.LightningNode,
 // if no payment attempts have been made.
 func (m *MissionControl) ResetHistory() {
 	m.Lock()
-	m.failedEdges = make(map[EdgeLocator]time.Time)
-	m.failedVertexes = make(map[route.Vertex]time.Time)
-	m.Unlock()
+	defer m.Unlock()
+
+	m.history = make(map[route.Vertex]*nodeHistory)
+
+	log.Debugf("Mission control history cleared")
+}
+
+// getEdgeProbability is expected to return the success probability of a payment
+// from fromNode along edge.
+func (m *MissionControl) getEdgeProbability(fromNode route.Vertex,
+	edge EdgeLocator, amt lnwire.MilliSatoshi) float64 {
+
+	m.Lock()
+	defer m.Unlock()
+
+	// Get the history for this node. If there is no history available,
+	// assume that it's success probability is a constant a priori
+	// probability. After the attempt new information becomes available to
+	// adjust this probability.
+	nodeHistory, ok := m.history[fromNode]
+	if !ok {
+		return m.cfg.AprioriHopProbability
+	}
+
+	return m.getEdgeProbabilityForNode(nodeHistory, edge.ChannelID, amt)
+}
+
+// getEdgeProbabilityForNode estimates the probability of successfully
+// traversing a channel based on the node history.
+func (m *MissionControl) getEdgeProbabilityForNode(nodeHistory *nodeHistory,
+	channelID uint64, amt lnwire.MilliSatoshi) float64 {
+
+	// Calculate the last failure of the given edge. A node failure is
+	// considered a failure that would have affected every edge. Therefore
+	// we insert a node level failure into the history of every channel.
+	lastFailure := nodeHistory.lastFail
+
+	// Take into account a minimum penalize amount. For balance errors, a
+	// failure may be reported with such a minimum to prevent too aggresive
+	// penalization. We only take into account a previous failure if the
+	// amount that we currently get the probability for is greater or equal
+	// than the minPenalizeAmt of the previous failure.
+	channelHistory, ok := nodeHistory.channelLastFail[channelID]
+	if ok && channelHistory.minPenalizeAmt <= amt {
+
+		// If there is both a node level failure recorded and a channel
+		// level failure is applicable too, we take the most recent of
+		// the two.
+		if lastFailure == nil ||
+			channelHistory.lastFail.After(*lastFailure) {
+
+			lastFailure = &channelHistory.lastFail
+		}
+	}
+
+	if lastFailure == nil {
+		return m.cfg.AprioriHopProbability
+	}
+
+	timeSinceLastFailure := m.now().Sub(*lastFailure)
+
+	// Calculate success probability. It is an exponential curve that brings
+	// the probability down to zero when a failure occurs. From there it
+	// recovers asymptotically back to the a priori probability. The rate at
+	// which this happens is controlled by the penaltyHalfLife parameter.
+	exp := -timeSinceLastFailure.Hours() / m.cfg.PenaltyHalfLife.Hours()
+	probability := m.cfg.AprioriHopProbability * (1 - math.Pow(2, exp))
+
+	return probability
+}
+
+// createHistoryIfNotExists returns the history for the given node. If the node
+// is yet unknown, it will create an empty history structure.
+func (m *MissionControl) createHistoryIfNotExists(vertex route.Vertex) *nodeHistory {
+	if node, ok := m.history[vertex]; ok {
+		return node
+	}
+
+	node := &nodeHistory{
+		channelLastFail: make(map[uint64]*channelHistory),
+	}
+	m.history[vertex] = node
+
+	return node
+}
+
+// reportVertexFailure reports a node level failure.
+func (m *MissionControl) reportVertexFailure(v route.Vertex) {
+	log.Debugf("Reporting vertex %v failure to Mission Control", v)
+
+	now := m.now()
+
+	m.Lock()
+	defer m.Unlock()
+
+	history := m.createHistoryIfNotExists(v)
+	history.lastFail = &now
+}
+
+// reportEdgeFailure reports a channel level failure.
+//
+// TODO(roasbeef): also add value attempted to send and capacity of channel
+func (m *MissionControl) reportEdgeFailure(failedEdge edge,
+	minPenalizeAmt lnwire.MilliSatoshi) {
+
+	log.Debugf("Reporting channel %v failure to Mission Control",
+		failedEdge.channel)
+
+	now := m.now()
+
+	m.Lock()
+	defer m.Unlock()
+
+	history := m.createHistoryIfNotExists(failedEdge.from)
+	history.channelLastFail[failedEdge.channel] = &channelHistory{
+		lastFail:       now,
+		minPenalizeAmt: minPenalizeAmt,
+	}
+}
+
+// GetHistorySnapshot takes a snapshot from the current mission control state
+// and actual probability estimates.
+func (m *MissionControl) GetHistorySnapshot() *MissionControlSnapshot {
+	m.Lock()
+	defer m.Unlock()
+
+	log.Debugf("Requesting history snapshot from mission control: "+
+		"node_count=%v", len(m.history))
+
+	nodes := make([]MissionControlNodeSnapshot, 0, len(m.history))
+
+	for v, h := range m.history {
+		channelSnapshot := make([]MissionControlChannelSnapshot, 0,
+			len(h.channelLastFail),
+		)
+
+		for id, lastFail := range h.channelLastFail {
+			// Show probability assuming amount meets min
+			// penalization amount.
+			prob := m.getEdgeProbabilityForNode(
+				h, id, lastFail.minPenalizeAmt,
+			)
+
+			channelSnapshot = append(channelSnapshot,
+				MissionControlChannelSnapshot{
+					ChannelID:      id,
+					LastFail:       lastFail.lastFail,
+					MinPenalizeAmt: lastFail.minPenalizeAmt,
+					SuccessProb:    prob,
+				},
+			)
+		}
+
+		otherProb := m.getEdgeProbabilityForNode(h, 0, 0)
+
+		nodes = append(nodes,
+			MissionControlNodeSnapshot{
+				Node:                 v,
+				LastFail:             h.lastFail,
+				OtherChanSuccessProb: otherProb,
+				Channels:             channelSnapshot,
+			},
+		)
+	}
+
+	snapshot := MissionControlSnapshot{
+		Nodes: nodes,
+	}
+
+	return &snapshot
 }

routing/missioncontrol_test.go

@@ -0,0 +1,81 @@
+package routing
+
+import (
+	"testing"
+	"time"
+
+	"github.com/lightningnetwork/lnd/lnwire"
+	"github.com/lightningnetwork/lnd/routing/route"
+)
+
+// TestMissionControl tests mission control probability estimation.
+func TestMissionControl(t *testing.T) {
+	now := testTime
+
+	mc := NewMissionControl(
+		nil, nil, nil, &MissionControlConfig{
+			PenaltyHalfLife:       30 * time.Minute,
+			AprioriHopProbability: 0.8,
+		},
+	)
+	mc.now = func() time.Time { return now }
+
+	testTime := time.Date(2018, time.January, 9, 14, 00, 00, 0, time.UTC)
+
+	testNode := route.Vertex{}
+	testEdge := edge{
+		channel: 123,
+	}
+
+	expectP := func(amt lnwire.MilliSatoshi, expected float64) {
+		t.Helper()
+
+		p := mc.getEdgeProbability(
+			testNode, EdgeLocator{ChannelID: testEdge.channel},
+			amt,
+		)
+		if p != expected {
+			t.Fatalf("unexpected probability %v", p)
+		}
+	}
+
+	// Initial probability is expected to be 1.
+	expectP(1000, 0.8)
+
+	// Expect probability to be zero after reporting the edge as failed.
+	mc.reportEdgeFailure(testEdge, 1000)
+	expectP(1000, 0)
+
+	// As we reported with a min penalization amt, a lower amt than reported
+	// should be unaffected.
+	expectP(500, 0.8)
+
+	// Edge decay started.
+	now = testTime.Add(30 * time.Minute)
+	expectP(1000, 0.4)
+
+	// Edge fails again, this time without a min penalization amt. The edge
+	// should be penalized regardless of amount.
+	mc.reportEdgeFailure(testEdge, 0)
+	expectP(1000, 0)
+	expectP(500, 0)
+
+	// Edge decay started.
+	now = testTime.Add(60 * time.Minute)
+	expectP(1000, 0.4)
+
+	// A node level failure should bring probability of every channel back
+	// to zero.
+	mc.reportVertexFailure(testNode)
+	expectP(1000, 0)
+
+	// Check whether history snapshot looks sane.
+	history := mc.GetHistorySnapshot()
+	if len(history.Nodes) != 1 {
+		t.Fatal("unexpected number of nodes")
+	}
+
+	if len(history.Nodes[0].Channels) != 1 {
+		t.Fatal("unexpected number of channels")
+	}
+}

routing/mock_test.go

@@ -112,10 +112,9 @@ func (m *mockPaymentSession) RequestRoute(payment *LightningPayment,
 
 func (m *mockPaymentSession) ReportVertexFailure(v route.Vertex) {}
 
-func (m *mockPaymentSession) ReportEdgeFailure(e *EdgeLocator) {}
+func (m *mockPaymentSession) ReportEdgeFailure(failedEdge edge, minPenalizeAmt lnwire.MilliSatoshi) {}
 
-func (m *mockPaymentSession) ReportEdgePolicyFailure(errSource route.Vertex, failedEdge *EdgeLocator) {
-}
+func (m *mockPaymentSession) ReportEdgePolicyFailure(failedEdge edge) {}
 
 type mockPayer struct {
 	sendResult       chan error

routing/pathfind.go

@@ -40,6 +40,22 @@ type pathFinder = func(g *graphParams, r *RestrictParams,
 	source, target route.Vertex, amt lnwire.MilliSatoshi) (
 	[]*channeldb.ChannelEdgePolicy, error)
 
+var (
+	// DefaultPaymentAttemptPenalty is the virtual cost in path finding weight
+	// units of executing a payment attempt that fails. It is used to trade
+	// off potentially better routes against their probability of
+	// succeeding.
+	DefaultPaymentAttemptPenalty = lnwire.NewMSatFromSatoshis(100)
+
+	// DefaultMinRouteProbability is the default minimum probability for routes
+	// returned from findPath.
+	DefaultMinRouteProbability = float64(0.01)
+
+	// DefaultAprioriHopProbability is the default a priori probability for
+	// a hop.
+	DefaultAprioriHopProbability = float64(0.95)
+)
+
 // edgePolicyWithSource is a helper struct to keep track of the source node
 // of a channel edge. ChannelEdgePolicy only contains to destination node
 // of the edge.
@@ -228,13 +244,10 @@ type graphParams struct {
 // RestrictParams wraps the set of restrictions passed to findPath that the
 // found path must adhere to.
 type RestrictParams struct {
-	// IgnoredNodes is an optional set of nodes that should be ignored if
-	// encountered during path finding.
-	IgnoredNodes map[route.Vertex]struct{}
-
-	// IgnoredEdges is an optional set of edges that should be ignored if
-	// encountered during path finding.
-	IgnoredEdges map[EdgeLocator]struct{}
+	// ProbabilitySource is a callback that is expected to return the
+	// success probability of traversing the channel from the node.
+	ProbabilitySource func(route.Vertex, EdgeLocator,
+		lnwire.MilliSatoshi) float64
 
 	// FeeLimit is a maximum fee amount allowed to be used on the path from
 	// the source to the target.
@@ -248,6 +261,16 @@ type RestrictParams struct {
 	// ctlv. After path finding is complete, the caller needs to increase
 	// all cltv expiry heights with the required final cltv delta.
 	CltvLimit *uint32
+
+	// PaymentAttemptPenalty is the virtual cost in path finding weight
+	// units of executing a payment attempt that fails. It is used to trade
+	// off potentially better routes against their probability of
+	// succeeding.
+	PaymentAttemptPenalty lnwire.MilliSatoshi
+
+	// MinProbability defines the minimum success probability of the
+	// returned route.
+	MinProbability float64
 }
 
 // findPath attempts to find a path from the source node within the
@@ -331,10 +354,11 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 	targetNode := &channeldb.LightningNode{PubKeyBytes: target}
 	distance[target] = nodeWithDist{
 		dist:            0,
+		weight:          0,
 		node:            targetNode,
 		amountToReceive: amt,
-		fee:             0,
 		incomingCltv:    0,
+		probability:     1,
 	}
 
 	// We'll use this map as a series of "next" hop pointers. So to get
@@ -342,15 +366,6 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 	// mapped to within `next`.
 	next := make(map[route.Vertex]*channeldb.ChannelEdgePolicy)
 
-	ignoredEdges := r.IgnoredEdges
-	if ignoredEdges == nil {
-		ignoredEdges = make(map[EdgeLocator]struct{})
-	}
-	ignoredNodes := r.IgnoredNodes
-	if ignoredNodes == nil {
-		ignoredNodes = make(map[route.Vertex]struct{})
-	}
-
 	// processEdge is a helper closure that will be used to make sure edges
 	// satisfy our specific requirements.
 	processEdge := func(fromNode *channeldb.LightningNode,
@@ -380,21 +395,26 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 			return
 		}
 
-		// If this vertex or edge has been black listed, then we'll
-		// skip exploring this edge.
-		if _, ok := ignoredNodes[fromVertex]; ok {
-			return
-		}
-
-		locator := newEdgeLocator(edge)
-		if _, ok := ignoredEdges[*locator]; ok {
-			return
-		}
-
+		// Calculate amount that the candidate node would have to sent
+		// out.
 		toNodeDist := distance[toNode]
-
 		amountToSend := toNodeDist.amountToReceive
 
+		// Request the success probability for this edge.
+		locator := newEdgeLocator(edge)
+		edgeProbability := r.ProbabilitySource(
+			fromVertex, *locator, amountToSend,
+		)
+
+		log.Tracef("path finding probability: fromnode=%v, chanid=%v, "+
+			"probability=%v", fromVertex, locator.ChannelID,
+			edgeProbability)
+
+		// If the probability is zero, there is no point in trying.
+		if edgeProbability == 0 {
+			return
+		}
+
 		// If the estimated bandwidth of the channel edge is not able
 		// to carry the amount that needs to be send, return.
 		if bandwidth < amountToSend {
@@ -453,19 +473,39 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 			return
 		}
 
+		// Calculate total probability of successfully reaching target
+		// by multiplying the probabilities. Both this edge and the rest
+		// of the route must succeed.
+		probability := toNodeDist.probability * edgeProbability
+
+		// If the probability is below the specified lower bound, we can
+		// abandon this direction. Adding further nodes can only lower
+		// the probability more.
+		if probability < r.MinProbability {
+			return
+		}
+
 		// By adding fromNode in the route, there will be an extra
 		// weight composed of the fee that this node will charge and
 		// the amount that will be locked for timeLockDelta blocks in
 		// the HTLC that is handed out to fromNode.
 		weight := edgeWeight(amountToReceive, fee, timeLockDelta)
 
-		// Compute the tentative distance to this new channel/edge
-		// which is the distance from our toNode to the target node
+		// Compute the tentative weight to this new channel/edge
+		// which is the weight from our toNode to the target node
 		// plus the weight of this edge.
-		tempDist := toNodeDist.dist + weight
+		tempWeight := toNodeDist.weight + weight
 
-		// If this new tentative distance is not better than the current
-		// best known distance to this node, return.
+		// Add an extra factor to the weight to take into account the
+		// probability.
+		tempDist := getProbabilityBasedDist(
+			tempWeight, probability, int64(r.PaymentAttemptPenalty),
+		)
+
+		// If the current best route is better than this candidate
+		// route, return. It is important to also return if the distance
+		// is equal, because otherwise the algorithm could run into an
+		// endless loop.
 		if tempDist >= distance[fromVertex].dist {
 			return
 		}
@@ -483,10 +523,11 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 		// map is populated with this edge.
 		distance[fromVertex] = nodeWithDist{
 			dist:            tempDist,
+			weight:          tempWeight,
 			node:            fromNode,
 			amountToReceive: amountToReceive,
-			fee:             fee,
 			incomingCltv:    incomingCltv,
+			probability:     probability,
 		}
 
 		next[fromVertex] = edge
@@ -614,5 +655,53 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 			"too many hops")
 	}
 
+	log.Debugf("Found route: probability=%v, hops=%v, fee=%v\n",
+		distance[source].probability, numEdges,
+		distance[source].amountToReceive-amt)
+
 	return pathEdges, nil
 }
+
+// getProbabilityBasedDist converts a weight into a distance that takes into
+// account the success probability and the (virtual) cost of a failed payment
+// attempt.
+//
+// Derivation:
+//
+// Suppose there are two routes A and B with fees Fa and Fb and success
+// probabilities Pa and Pb.
+//
+// Is the expected cost of trying route A first and then B lower than trying the
+// other way around?
+//
+// The expected cost of A-then-B is: Pa*Fa + (1-Pa)*Pb*(c+Fb)
+//
+// The expected cost of B-then-A is: Pb*Fb + (1-Pb)*Pa*(c+Fa)
+//
+// In these equations, the term representing the case where both A and B fail is
+// left out because its value would be the same in both cases.
+//
+// Pa*Fa + (1-Pa)*Pb*(c+Fb) < Pb*Fb + (1-Pb)*Pa*(c+Fa)
+//
+// Pa*Fa + Pb*c + Pb*Fb - Pa*Pb*c - Pa*Pb*Fb < Pb*Fb + Pa*c + Pa*Fa - Pa*Pb*c - Pa*Pb*Fa
+//
+// Removing terms that cancel out:
+// Pb*c - Pa*Pb*Fb < Pa*c - Pa*Pb*Fa
+//
+// Divide by Pa*Pb:
+// c/Pa - Fb < c/Pb - Fa
+//
+// Move terms around:
+// Fa + c/Pa < Fb + c/Pb
+//
+// So the value of F + c/P can be used to compare routes.
+func getProbabilityBasedDist(weight int64, probability float64, penalty int64) int64 {
+	// Clamp probability to prevent overflow.
+	const minProbability = 0.00001
+
+	if probability < minProbability {
+		return infinity
+	}
+
+	return weight + int64(float64(penalty)/probability)
+}

routing/pathfind_test.go

@@ -52,7 +52,8 @@ const (
 
 var (
 	noRestrictions = &RestrictParams{
-		FeeLimit: noFeeLimit,
+		FeeLimit:          noFeeLimit,
+		ProbabilitySource: noProbabilitySource,
 	}
 )
 
@@ -72,6 +73,12 @@ var (
 	}
 )
 
+// noProbabilitySource is used in testing to return the same probability 1 for
+// all edges.
+func noProbabilitySource(route.Vertex, EdgeLocator, lnwire.MilliSatoshi) float64 {
+	return 1
+}
+
 // testGraph is the struct which corresponds to the JSON format used to encode
 // graphs within the files in the testdata directory.
 //
@@ -477,6 +484,16 @@ func createTestGraphFromChannels(testChannels []*testChannel) (*testGraphInstanc
 			Index: 0,
 		}
 
+		// Sort nodes
+		node1 := testChannel.Node1
+		node2 := testChannel.Node2
+		node1Vertex := aliasMap[node1.Alias]
+		node2Vertex := aliasMap[node2.Alias]
+		if bytes.Compare(node1Vertex[:], node2Vertex[:]) == 1 {
+			node1, node2 = node2, node1
+			node1Vertex, node2Vertex = node2Vertex, node1Vertex
+		}
+
 		// We first insert the existence of the edge between the two
 		// nodes.
 		edgeInfo := channeldb.ChannelEdgeInfo{
@@ -485,10 +502,10 @@ func createTestGraphFromChannels(testChannels []*testChannel) (*testGraphInstanc
 			ChannelPoint: *fundingPoint,
 			Capacity:     testChannel.Capacity,
 
-			NodeKey1Bytes:    aliasMap[testChannel.Node1.Alias],
-			BitcoinKey1Bytes: aliasMap[testChannel.Node1.Alias],
-			NodeKey2Bytes:    aliasMap[testChannel.Node2.Alias],
-			BitcoinKey2Bytes: aliasMap[testChannel.Node2.Alias],
+			NodeKey1Bytes:    node1Vertex,
+			BitcoinKey1Bytes: node1Vertex,
+			NodeKey2Bytes:    node2Vertex,
+			BitcoinKey2Bytes: node2Vertex,
 		}
 
 		err = graph.AddChannelEdge(&edgeInfo)
@@ -510,12 +527,12 @@ func createTestGraphFromChannels(testChannels []*testChannel) (*testGraphInstanc
 				MessageFlags:              msgFlags,
 				ChannelFlags:              channelFlags,
 				ChannelID:                 channelID,
-				LastUpdate:                testChannel.Node1.LastUpdate,
-				TimeLockDelta:             testChannel.Node1.Expiry,
-				MinHTLC:                   testChannel.Node1.MinHTLC,
-				MaxHTLC:                   testChannel.Node1.MaxHTLC,
-				FeeBaseMSat:               testChannel.Node1.FeeBaseMsat,
-				FeeProportionalMillionths: testChannel.Node1.FeeRate,
+				LastUpdate:                node1.LastUpdate,
+				TimeLockDelta:             node1.Expiry,
+				MinHTLC:                   node1.MinHTLC,
+				MaxHTLC:                   node1.MaxHTLC,
+				FeeBaseMSat:               node1.FeeBaseMsat,
+				FeeProportionalMillionths: node1.FeeRate,
 			}
 			if err := graph.UpdateEdgePolicy(edgePolicy); err != nil {
 				return nil, err
@@ -536,12 +553,12 @@ func createTestGraphFromChannels(testChannels []*testChannel) (*testGraphInstanc
 				MessageFlags:              msgFlags,
 				ChannelFlags:              channelFlags,
 				ChannelID:                 channelID,
-				LastUpdate:                testChannel.Node2.LastUpdate,
-				TimeLockDelta:             testChannel.Node2.Expiry,
-				MinHTLC:                   testChannel.Node2.MinHTLC,
-				MaxHTLC:                   testChannel.Node2.MaxHTLC,
-				FeeBaseMSat:               testChannel.Node2.FeeBaseMsat,
-				FeeProportionalMillionths: testChannel.Node2.FeeRate,
+				LastUpdate:                node2.LastUpdate,
+				TimeLockDelta:             node2.Expiry,
+				MinHTLC:                   node2.MinHTLC,
+				MaxHTLC:                   node2.MaxHTLC,
+				FeeBaseMSat:               node2.FeeBaseMsat,
+				FeeProportionalMillionths: node2.FeeRate,
 			}
 			if err := graph.UpdateEdgePolicy(edgePolicy); err != nil {
 				return nil, err
@@ -625,9 +642,7 @@ func TestFindLowestFeePath(t *testing.T) {
 		&graphParams{
 			graph: testGraphInstance.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, paymentAmt,
 	)
 	if err != nil {
@@ -766,7 +781,8 @@ func testBasicGraphPathFindingCase(t *testing.T, graphInstance *testGraphInstanc
 			graph: graphInstance.graph,
 		},
 		&RestrictParams{
-			FeeLimit: test.feeLimit,
+			FeeLimit:          test.feeLimit,
+			ProbabilitySource: noProbabilitySource,
 		},
 		sourceNode.PubKeyBytes, target, paymentAmt,
 	)
@@ -934,9 +950,7 @@ func TestPathFindingWithAdditionalEdges(t *testing.T) {
 			graph:           graph.graph,
 			additionalEdges: additionalEdges,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, doge.PubKeyBytes, paymentAmt,
 	)
 	if err != nil {
@@ -1190,9 +1204,7 @@ func TestNewRoutePathTooLong(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, paymentAmt,
 	)
 	if err != nil {
@@ -1206,9 +1218,7 @@ func TestNewRoutePathTooLong(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, paymentAmt,
 	)
 	if err == nil {
@@ -1248,9 +1258,7 @@ func TestPathNotAvailable(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, unknownNode, 100,
 	)
 	if !IsError(err, ErrNoPathFound) {
@@ -1287,9 +1295,7 @@ func TestPathInsufficientCapacity(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if !IsError(err, ErrNoPathFound) {
@@ -1322,9 +1328,7 @@ func TestRouteFailMinHTLC(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if !IsError(err, ErrNoPathFound) {
@@ -1382,9 +1386,7 @@ func TestRouteFailMaxHTLC(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if err != nil {
@@ -1406,9 +1408,7 @@ func TestRouteFailMaxHTLC(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if !IsError(err, ErrNoPathFound) {
@@ -1443,9 +1443,7 @@ func TestRouteFailDisabledEdge(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if err != nil {
@@ -1473,9 +1471,7 @@ func TestRouteFailDisabledEdge(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if err != nil {
@@ -1500,9 +1496,7 @@ func TestRouteFailDisabledEdge(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if !IsError(err, ErrNoPathFound) {
@@ -1536,9 +1530,7 @@ func TestPathSourceEdgesBandwidth(t *testing.T) {
 		&graphParams{
 			graph: graph.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if err != nil {
@@ -1562,9 +1554,7 @@ func TestPathSourceEdgesBandwidth(t *testing.T) {
 			graph:          graph.graph,
 			bandwidthHints: bandwidths,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if !IsError(err, ErrNoPathFound) {
@@ -1582,9 +1572,7 @@ func TestPathSourceEdgesBandwidth(t *testing.T) {
 			graph:          graph.graph,
 			bandwidthHints: bandwidths,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if err != nil {
@@ -1615,9 +1603,7 @@ func TestPathSourceEdgesBandwidth(t *testing.T) {
 			graph:          graph.graph,
 			bandwidthHints: bandwidths,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, payAmt,
 	)
 	if err != nil {
@@ -1906,6 +1892,7 @@ func TestRestrictOutgoingChannel(t *testing.T) {
 		&RestrictParams{
 			FeeLimit:          noFeeLimit,
 			OutgoingChannelID: &outgoingChannelID,
+			ProbabilitySource: noProbabilitySource,
 		},
 		sourceVertex, target, paymentAmt,
 	)
@@ -1982,9 +1969,6 @@ func testCltvLimit(t *testing.T, limit uint32, expectedChannel uint64) {
 	}
 	sourceVertex := route.Vertex(sourceNode.PubKeyBytes)
 
-	ignoredEdges := make(map[EdgeLocator]struct{})
-	ignoredVertexes := make(map[route.Vertex]struct{})
-
 	paymentAmt := lnwire.NewMSatFromSatoshis(100)
 	target := testGraphInstance.aliasMap["target"]
 
@@ -1999,10 +1983,9 @@ func testCltvLimit(t *testing.T, limit uint32, expectedChannel uint64) {
 			graph: testGraphInstance.graph,
 		},
 		&RestrictParams{
-			IgnoredNodes: ignoredVertexes,
-			IgnoredEdges: ignoredEdges,
-			FeeLimit:     noFeeLimit,
-			CltvLimit:    cltvLimit,
+			FeeLimit:          noFeeLimit,
+			CltvLimit:         cltvLimit,
+			ProbabilitySource: noProbabilitySource,
 		},
 		sourceVertex, target, paymentAmt,
 	)
@@ -2035,3 +2018,170 @@ func testCltvLimit(t *testing.T, limit uint32, expectedChannel uint64) {
 			route.Hops[0].ChannelID)
 	}
 }
+
+// TestProbabilityRouting asserts that path finding not only takes into account
+// fees but also success probability.
+func TestProbabilityRouting(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name           string
+		p10, p11, p20  float64
+		minProbability float64
+		expectedChan   uint64
+	}{
+		// Test two variations with probabilities that should multiply
+		// to the same total route probability. In both cases the three
+		// hop route should be the best route. The three hop route has a
+		// probability of 0.5 * 0.8 = 0.4. The fee is 5 (chan 10) + 8
+		// (chan 11) = 13. Path finding distance should work out to: 13
+		// + 10 (attempt penalty) / 0.4 = 38. The two hop route is 25 +
+		// 10 / 0.7 = 39.
+		{
+			name: "three hop 1",
+			p10:  0.8, p11: 0.5, p20: 0.7,
+			minProbability: 0.1,
+			expectedChan:   10,
+		},
+		{
+			name: "three hop 2",
+			p10:  0.5, p11: 0.8, p20: 0.7,
+			minProbability: 0.1,
+			expectedChan:   10,
+		},
+
+		// If the probability of the two hop route is increased, its
+		// distance becomes 25 + 10 / 0.85 = 37. This is less than the
+		// three hop route with its distance 38. So with an attempt
+		// penalty of 10, the higher fee route is chosen because of the
+		// compensation for success probability.
+		{
+			name: "two hop higher cost",
+			p10:  0.5, p11: 0.8, p20: 0.85,
+			minProbability: 0.1,
+			expectedChan:   20,
+		},
+
+		// If the same probabilities are used with a probability lower bound of
+		// 0.5, we expect the three hop route with probability 0.4 to be
+		// excluded and the two hop route to be picked.
+		{
+			name: "probability limit",
+			p10:  0.8, p11: 0.5, p20: 0.7,
+			minProbability: 0.5,
+			expectedChan:   20,
+		},
+
+		// With a probability limit above the probability of both routes, we
+		// expect no route to be returned. This expectation is signaled by using
+		// expected channel 0.
+		{
+			name: "probability limit no routes",
+			p10:  0.8, p11: 0.5, p20: 0.7,
+			minProbability: 0.8,
+			expectedChan:   0,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			testProbabilityRouting(
+				t, tc.p10, tc.p11, tc.p20,
+				tc.minProbability, tc.expectedChan,
+			)
+		})
+	}
+}
+
+func testProbabilityRouting(t *testing.T, p10, p11, p20, minProbability float64,
+	expectedChan uint64) {
+
+	t.Parallel()
+
+	// Set up a test graph with two possible paths to the target: a three
+	// hop path (via channels 10 and 11) and a two hop path (via channel
+	// 20).
+	testChannels := []*testChannel{
+		symmetricTestChannel("roasbeef", "a1", 100000, &testChannelPolicy{}),
+		symmetricTestChannel("roasbeef", "b", 100000, &testChannelPolicy{}),
+		symmetricTestChannel("a1", "a2", 100000, &testChannelPolicy{
+			Expiry:      144,
+			FeeBaseMsat: lnwire.NewMSatFromSatoshis(5),
+			MinHTLC:     1,
+		}, 10),
+		symmetricTestChannel("a2", "target", 100000, &testChannelPolicy{
+			Expiry:      144,
+			FeeBaseMsat: lnwire.NewMSatFromSatoshis(8),
+			MinHTLC:     1,
+		}, 11),
+		symmetricTestChannel("b", "target", 100000, &testChannelPolicy{
+			Expiry:      100,
+			FeeBaseMsat: lnwire.NewMSatFromSatoshis(25),
+			MinHTLC:     1,
+		}, 20),
+	}
+
+	testGraphInstance, err := createTestGraphFromChannels(testChannels)
+	if err != nil {
+		t.Fatalf("unable to create graph: %v", err)
+	}
+	defer testGraphInstance.cleanUp()
+
+	sourceNode, err := testGraphInstance.graph.SourceNode()
+	if err != nil {
+		t.Fatalf("unable to fetch source node: %v", err)
+	}
+	sourceVertex := route.Vertex(sourceNode.PubKeyBytes)
+
+	paymentAmt := lnwire.NewMSatFromSatoshis(100)
+	target := testGraphInstance.aliasMap["target"]
+
+	// Configure a probability source with the test parameters.
+	probabilitySource := func(node route.Vertex, edge EdgeLocator,
+		amt lnwire.MilliSatoshi) float64 {
+
+		if amt == 0 {
+			t.Fatal("expected non-zero amount")
+		}
+
+		switch edge.ChannelID {
+		case 10:
+			return p10
+		case 11:
+			return p11
+		case 20:
+			return p20
+		default:
+			return 1
+		}
+	}
+
+	path, err := findPath(
+		&graphParams{
+			graph: testGraphInstance.graph,
+		},
+		&RestrictParams{
+			FeeLimit:              noFeeLimit,
+			ProbabilitySource:     probabilitySource,
+			PaymentAttemptPenalty: lnwire.NewMSatFromSatoshis(10),
+			MinProbability:        minProbability,
+		},
+		sourceVertex, target, paymentAmt,
+	)
+	if expectedChan == 0 {
+		if err == nil || !IsError(err, ErrNoPathFound) {
+			t.Fatalf("expected no path found, but got %v", err)
+		}
+		return
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Assert that the route passes through the expected channel.
+	if path[1].ChannelID != expectedChan {
+		t.Fatalf("expected route to pass through channel %v, "+
+			"but channel %v was selected instead", expectedChan,
+			path[1].ChannelID)
+	}
+}

routing/payment_session.go

@@ -2,7 +2,6 @@ package routing
 
 import (
 	"fmt"
-	"time"
 
 	"github.com/lightningnetwork/lnd/channeldb"
 	"github.com/lightningnetwork/lnd/lnwire"
@@ -24,11 +23,12 @@ type PaymentSession interface {
 	// route.
 	ReportVertexFailure(v route.Vertex)
 
-	// ReportEdgeFailure reports to the PaymentSession that the passed
-	// channel failed to route the previous payment attempt. The
-	// PaymentSession will use this information to produce a better next
-	// route.
-	ReportEdgeFailure(e *EdgeLocator)
+	// ReportEdgeFailure reports to the PaymentSession that the passed edge
+	// failed to route the previous payment attempt. A minimum penalization
+	// amount is included to attenuate the failure. This is set to a
+	// non-zero value for channel balance failures. The PaymentSession will
+	// use this information to produce a better next route.
+	ReportEdgeFailure(failedEdge edge, minPenalizeAmt lnwire.MilliSatoshi)
 
 	// ReportEdgePolicyFailure reports to the PaymentSession that we
 	// received a failure message that relates to a channel policy. For
@@ -36,7 +36,7 @@ type PaymentSession interface {
 	// keep the edge included in the next attempted route. The
 	// PaymentSession will use this information to produce a better next
 	// route.
-	ReportEdgePolicyFailure(errSource route.Vertex, failedEdge *EdgeLocator)
+	ReportEdgePolicyFailure(failedEdge edge)
 }
 
 // paymentSession is used during an HTLC routings session to prune the local
@@ -48,8 +48,6 @@ type PaymentSession interface {
 // loop if payment attempts take long enough. An additional set of edges can
 // also be provided to assist in reaching the payment's destination.
 type paymentSession struct {
-	pruneViewSnapshot graphPruneView
-
 	additionalEdges map[route.Vertex][]*channeldb.ChannelEdgePolicy
 
 	bandwidthHints map[uint64]lnwire.MilliSatoshi
@@ -58,7 +56,7 @@ type paymentSession struct {
 	// source of policy related routing failures during this payment attempt.
 	// We'll use this map to prune out channels when the first error may not
 	// require pruning, but any subsequent ones do.
-	errFailedPolicyChans map[EdgeLocator]struct{}
+	errFailedPolicyChans map[nodeChannel]struct{}
 
 	mc *MissionControl
 
@@ -80,17 +78,7 @@ var _ PaymentSession = (*paymentSession)(nil)
 //
 // NOTE: Part of the PaymentSession interface.
 func (p *paymentSession) ReportVertexFailure(v route.Vertex) {
-	log.Debugf("Reporting vertex %v failure to Mission Control", v)
-
-	// First, we'll add the failed vertex to our local prune view snapshot.
-	p.pruneViewSnapshot.vertexes[v] = struct{}{}
-
-	// With the vertex added, we'll now report back to the global prune
-	// view, with this new piece of information so it can be utilized for
-	// new payment sessions.
-	p.mc.Lock()
-	p.mc.failedVertexes[v] = time.Now()
-	p.mc.Unlock()
+	p.mc.reportVertexFailure(v)
 }
 
 // ReportEdgeFailure adds a channel to the graph prune view. The time the
@@ -102,18 +90,10 @@ func (p *paymentSession) ReportVertexFailure(v route.Vertex) {
 // TODO(roasbeef): also add value attempted to send and capacity of channel
 //
 // NOTE: Part of the PaymentSession interface.
-func (p *paymentSession) ReportEdgeFailure(e *EdgeLocator) {
-	log.Debugf("Reporting edge %v failure to Mission Control", e)
+func (p *paymentSession) ReportEdgeFailure(failedEdge edge,
+	minPenalizeAmt lnwire.MilliSatoshi) {
 
-	// First, we'll add the failed edge to our local prune view snapshot.
-	p.pruneViewSnapshot.edges[*e] = struct{}{}
-
-	// With the edge added, we'll now report back to the global prune view,
-	// with this new piece of information so it can be utilized for new
-	// payment sessions.
-	p.mc.Lock()
-	p.mc.failedEdges[*e] = time.Now()
-	p.mc.Unlock()
+	p.mc.reportEdgeFailure(failedEdge, minPenalizeAmt)
 }
 
 // ReportEdgePolicyFailure handles a failure message that relates to a
@@ -124,23 +104,28 @@ func (p *paymentSession) ReportEdgeFailure(e *EdgeLocator) {
 // new channel updates.
 //
 // NOTE: Part of the PaymentSession interface.
-func (p *paymentSession) ReportEdgePolicyFailure(
-	errSource route.Vertex, failedEdge *EdgeLocator) {
+//
+// TODO(joostjager): Move this logic into global mission control.
+func (p *paymentSession) ReportEdgePolicyFailure(failedEdge edge) {
+	key := nodeChannel{
+		node:    failedEdge.from,
+		channel: failedEdge.channel,
+	}
 
 	// Check to see if we've already reported a policy related failure for
 	// this channel. If so, then we'll prune out the vertex.
-	_, ok := p.errFailedPolicyChans[*failedEdge]
+	_, ok := p.errFailedPolicyChans[key]
 	if ok {
 		// TODO(joostjager): is this aggressive pruning still necessary?
 		// Just pruning edges may also work unless there is a huge
 		// number of failing channels from that node?
-		p.ReportVertexFailure(errSource)
+		p.ReportVertexFailure(key.node)
 
 		return
 	}
 
 	// Finally, we'll record a policy failure from this node and move on.
-	p.errFailedPolicyChans[*failedEdge] = struct{}{}
+	p.errFailedPolicyChans[key] = struct{}{}
 }
 
 // RequestRoute returns a route which is likely to be capable for successfully
@@ -169,15 +154,6 @@ func (p *paymentSession) RequestRoute(payment *LightningPayment,
 		return nil, fmt.Errorf("pre-built route already tried")
 	}
 
-	// Otherwise we actually need to perform path finding, so we'll obtain
-	// our current prune view snapshot. This view will only ever grow
-	// during the duration of this payment session, never shrinking.
-	pruneView := p.pruneViewSnapshot
-
-	log.Debugf("Mission Control session using prune view of %v "+
-		"edges, %v vertexes", len(pruneView.edges),
-		len(pruneView.vertexes))
-
 	// If a route cltv limit was specified, we need to subtract the final
 	// delta before passing it into path finding. The optimal path is
 	// independent of the final cltv delta and the path finding algorithm is
@@ -200,11 +176,12 @@ func (p *paymentSession) RequestRoute(payment *LightningPayment,
 			bandwidthHints:  p.bandwidthHints,
 		},
 		&RestrictParams{
-			IgnoredNodes:      pruneView.vertexes,
-			IgnoredEdges:      pruneView.edges,
-			FeeLimit:          payment.FeeLimit,
-			OutgoingChannelID: payment.OutgoingChannelID,
-			CltvLimit:         cltvLimit,
+			ProbabilitySource:     p.mc.getEdgeProbability,
+			FeeLimit:              payment.FeeLimit,
+			OutgoingChannelID:     payment.OutgoingChannelID,
+			CltvLimit:             cltvLimit,
+			PaymentAttemptPenalty: p.mc.cfg.PaymentAttemptPenalty,
+			MinProbability:        p.mc.cfg.MinRouteProbability,
 		},
 		p.mc.selfNode.PubKeyBytes, payment.Target,
 		payment.Amount,
@@ -227,3 +204,9 @@ func (p *paymentSession) RequestRoute(payment *LightningPayment,
 
 	return route, err
 }
+
+// nodeChannel is a combination of the node pubkey and one of its channels.
+type nodeChannel struct {
+	node    route.Vertex
+	channel uint64
+}

routing/payment_session_test.go

@@ -35,9 +35,9 @@ func TestRequestRoute(t *testing.T) {
 	session := &paymentSession{
 		mc: &MissionControl{
 			selfNode: &channeldb.LightningNode{},
+			cfg:      &MissionControlConfig{},
 		},
-		pruneViewSnapshot: graphPruneView{},
-		pathFinder:        findPath,
+		pathFinder: findPath,
 	}
 
 	cltvLimit := uint32(30)

routing/route/route.go

@@ -3,6 +3,8 @@ package route
 import (
 	"encoding/binary"
 	"fmt"
+	"strconv"
+	"strings"
 
 	"github.com/btcsuite/btcd/btcec"
 	sphinx "github.com/lightningnetwork/lightning-onion"
@@ -180,3 +182,20 @@ func (r *Route) ToSphinxPath() (*sphinx.PaymentPath, error) {
 
 	return &path, nil
 }
+
+// String returns a human readable representation of the route.
+func (r *Route) String() string {
+	var b strings.Builder
+
+	for i, hop := range r.Hops {
+		if i > 0 {
+			b.WriteString(",")
+		}
+		b.WriteString(strconv.FormatUint(hop.ChannelID, 10))
+	}
+
+	return fmt.Sprintf("amt=%v, fees=%v, tl=%v, chans=%v",
+		r.TotalAmount-r.TotalFees(), r.TotalFees(), r.TotalTimeLock,
+		b.String(),
+	)
+}

routing/router.go

@@ -319,6 +319,13 @@ func (e *EdgeLocator) String() string {
 	return fmt.Sprintf("%v:%v", e.ChannelID, e.Direction)
 }
 
+// edge is a combination of a channel and the node pubkeys of both of its
+// endpoints.
+type edge struct {
+	from, to route.Vertex
+	channel  uint64
+}
+
 // ChannelRouter is the layer 3 router within the Lightning stack. Below the
 // ChannelRouter is the HtlcSwitch, and below that is the Bitcoin blockchain
 // itself. The primary role of the ChannelRouter is to respond to queries for
@@ -1598,7 +1605,9 @@ type LightningPayment struct {
 // will be returned which describes the path the successful payment traversed
 // within the network to reach the destination. Additionally, the payment
 // preimage will also be returned.
-func (r *ChannelRouter) SendPayment(payment *LightningPayment) ([32]byte, *route.Route, error) {
+func (r *ChannelRouter) SendPayment(payment *LightningPayment) ([32]byte,
+	*route.Route, error) {
+
 	// Before starting the HTLC routing attempt, we'll create a fresh
 	// payment session which will report our errors back to mission
 	// control.
@@ -1772,7 +1781,9 @@ func (r *ChannelRouter) processSendError(paySession PaymentSession,
 
 	// Always determine chan id ourselves, because a channel
 	// update with id may not be available.
-	failedEdge, err := getFailedEdge(rt, route.Vertex(errVertex))
+	failedEdge, failedAmt, err := getFailedEdge(
+		rt, route.Vertex(errVertex),
+	)
 	if err != nil {
 		return true
 	}
@@ -1804,13 +1815,11 @@ func (r *ChannelRouter) processSendError(paySession PaymentSession,
 		// update to fail?
 		if !updateOk {
 			paySession.ReportEdgeFailure(
-				failedEdge,
+				failedEdge, 0,
 			)
 		}
 
-		paySession.ReportEdgePolicyFailure(
-			route.NewVertex(errSource), failedEdge,
-		)
+		paySession.ReportEdgePolicyFailure(failedEdge)
 	}
 
 	switch onionErr := fErr.FailureMessage.(type) {
@@ -1901,7 +1910,7 @@ func (r *ChannelRouter) processSendError(paySession PaymentSession,
 	// the update and continue.
 	case *lnwire.FailChannelDisabled:
 		r.applyChannelUpdate(&onionErr.Update, errSource)
-		paySession.ReportEdgeFailure(failedEdge)
+		paySession.ReportEdgeFailure(failedEdge, 0)
 		return false
 
 	// It's likely that the outgoing channel didn't have
@@ -1909,7 +1918,7 @@ func (r *ChannelRouter) processSendError(paySession PaymentSession,
 	// now, and continue onwards with our path finding.
 	case *lnwire.FailTemporaryChannelFailure:
 		r.applyChannelUpdate(onionErr.Update, errSource)
-		paySession.ReportEdgeFailure(failedEdge)
+		paySession.ReportEdgeFailure(failedEdge, failedAmt)
 		return false
 
 	// If the send fail due to a node not having the
@@ -1934,7 +1943,7 @@ func (r *ChannelRouter) processSendError(paySession PaymentSession,
 	// returning errors in order to attempt to black list
 	// another node.
 	case *lnwire.FailUnknownNextPeer:
-		paySession.ReportEdgeFailure(failedEdge)
+		paySession.ReportEdgeFailure(failedEdge, 0)
 		return false
 
 	// If the node wasn't able to forward for which ever
@@ -1965,14 +1974,12 @@ func (r *ChannelRouter) processSendError(paySession PaymentSession,
 	// we'll prune the channel in both directions and
 	// continue with the rest of the routes.
 	case *lnwire.FailPermanentChannelFailure:
-		paySession.ReportEdgeFailure(&EdgeLocator{
-			ChannelID: failedEdge.ChannelID,
-			Direction: 0,
-		})
-		paySession.ReportEdgeFailure(&EdgeLocator{
-			ChannelID: failedEdge.ChannelID,
-			Direction: 1,
-		})
+		paySession.ReportEdgeFailure(failedEdge, 0)
+		paySession.ReportEdgeFailure(edge{
+			from:    failedEdge.to,
+			to:      failedEdge.from,
+			channel: failedEdge.channel,
+		}, 0)
 		return false
 
 	default:
@@ -1982,12 +1989,14 @@ func (r *ChannelRouter) processSendError(paySession PaymentSession,
 
 // getFailedEdge tries to locate the failing channel given a route and the
 // pubkey of the node that sent the error. It will assume that the error is
-// associated with the outgoing channel of the error node.
-func getFailedEdge(route *route.Route, errSource route.Vertex) (
-	*EdgeLocator, error) {
+// associated with the outgoing channel of the error node. As a second result,
+// it returns the amount sent over the edge.
+func getFailedEdge(route *route.Route, errSource route.Vertex) (edge,
+	lnwire.MilliSatoshi, error) {
 
 	hopCount := len(route.Hops)
 	fromNode := route.SourcePubKey
+	amt := route.TotalAmount
 	for i, hop := range route.Hops {
 		toNode := hop.PubKeyBytes
 
@@ -2006,17 +2015,18 @@ func getFailedEdge(route *route.Route, errSource route.Vertex) (
 		// If the errSource is the final hop, we assume that the failing
 		// channel is the incoming channel.
 		if errSource == fromNode || finalHopFailing {
-			return newEdgeLocatorByPubkeys(
-				hop.ChannelID,
-				&fromNode,
-				&toNode,
-			), nil
+			return edge{
+				from:    fromNode,
+				to:      toNode,
+				channel: hop.ChannelID,
+			}, amt, nil
 		}
 
 		fromNode = toNode
+		amt = hop.AmtToForward
 	}
 
-	return nil, fmt.Errorf("cannot find error source node in route")
+	return edge{}, 0, fmt.Errorf("cannot find error source node in route")
 }
 
 // applyChannelUpdate validates a channel update and if valid, applies it to the

routing/router_test.go

@@ -95,6 +95,12 @@ func createTestCtxFromGraphInstance(startingHeight uint32, graphInstance *testGr
 		func(e *channeldb.ChannelEdgeInfo) lnwire.MilliSatoshi {
 			return lnwire.NewMSatFromSatoshis(e.Capacity)
 		},
+		&MissionControlConfig{
+			MinRouteProbability:   0.01,
+			PaymentAttemptPenalty: 100,
+			PenaltyHalfLife:       time.Hour,
+			AprioriHopProbability: 0.9,
+		},
 	)
 	router, err := New(Config{
 		Graph:              graphInstance.graph,
@@ -201,7 +207,8 @@ func TestFindRoutesWithFeeLimit(t *testing.T) {
 	target := ctx.aliases["sophon"]
 	paymentAmt := lnwire.NewMSatFromSatoshis(100)
 	restrictions := &RestrictParams{
-		FeeLimit: lnwire.NewMSatFromSatoshis(10),
+		FeeLimit:          lnwire.NewMSatFromSatoshis(10),
+		ProbabilitySource: noProbabilitySource,
 	}
 
 	route, err := ctx.router.FindRoute(
@@ -2198,9 +2205,7 @@ func TestFindPathFeeWeighting(t *testing.T) {
 		&graphParams{
 			graph: ctx.graph,
 		},
-		&RestrictParams{
-			FeeLimit: noFeeLimit,
-		},
+		noRestrictions,
 		sourceNode.PubKeyBytes, target, amt,
 	)
 	if err != nil {