routing: use probability source in path finding

This PR replaces the previously used edge and node ignore lists in path
finding by a probability based system. It modifies path finding so that
it not only compares routes on fee and time lock, but also takes route
success probability into account.

Allowing routes to be compared based on success probability is achieved
by introducing a 'virtual' cost of a payment attempt and using that to
translate probability into another cost factor.

routing/pathfind.go

@@ -40,6 +40,14 @@ type pathFinder = func(g *graphParams, r *RestrictParams,
 	source, target route.Vertex, amt lnwire.MilliSatoshi) (
 	[]*channeldb.ChannelEdgePolicy, error)
 
+var (
+	// DefaultPaymentAttemptPenalty is the virtual cost in path finding weight
+	// units of executing a payment attempt that fails. It is used to trade
+	// off potentially better routes against their probability of
+	// succeeding.
+	DefaultPaymentAttemptPenalty = lnwire.NewMSatFromSatoshis(100)
+)
+
 // edgePolicyWithSource is a helper struct to keep track of the source node
 // of a channel edge. ChannelEdgePolicy only contains to destination node
 // of the edge.
@@ -228,13 +236,9 @@ type graphParams struct {
 // RestrictParams wraps the set of restrictions passed to findPath that the
 // found path must adhere to.
 type RestrictParams struct {
-	// IgnoredNodes is an optional set of nodes that should be ignored if
-	// encountered during path finding.
-	IgnoredNodes map[route.Vertex]struct{}
-
-	// IgnoredEdges is an optional set of edges that should be ignored if
-	// encountered during path finding.
-	IgnoredEdges map[EdgeLocator]struct{}
+	// ProbabilitySource is a callback that is expected to return the
+	// success probability of traversing the channel from the node.
+	ProbabilitySource func(route.Vertex, EdgeLocator) float64
 
 	// FeeLimit is a maximum fee amount allowed to be used on the path from
 	// the source to the target.
@@ -248,6 +252,12 @@ type RestrictParams struct {
 	// ctlv. After path finding is complete, the caller needs to increase
 	// all cltv expiry heights with the required final cltv delta.
 	CltvLimit *uint32
+
+	// PaymentAttemptPenalty is the virtual cost in path finding weight
+	// units of executing a payment attempt that fails. It is used to trade
+	// off potentially better routes against their probability of
+	// succeeding.
+	PaymentAttemptPenalty lnwire.MilliSatoshi
 }
 
 // findPath attempts to find a path from the source node within the
@@ -331,10 +341,11 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 	targetNode := &channeldb.LightningNode{PubKeyBytes: target}
 	distance[target] = nodeWithDist{
 		dist:            0,
+		weight:          0,
 		node:            targetNode,
 		amountToReceive: amt,
-		fee:             0,
 		incomingCltv:    0,
+		probability:     1,
 	}
 
 	// We'll use this map as a series of "next" hop pointers. So to get
@@ -342,15 +353,6 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 	// mapped to within `next`.
 	next := make(map[route.Vertex]*channeldb.ChannelEdgePolicy)
 
-	ignoredEdges := r.IgnoredEdges
-	if ignoredEdges == nil {
-		ignoredEdges = make(map[EdgeLocator]struct{})
-	}
-	ignoredNodes := r.IgnoredNodes
-	if ignoredNodes == nil {
-		ignoredNodes = make(map[route.Vertex]struct{})
-	}
-
 	// processEdge is a helper closure that will be used to make sure edges
 	// satisfy our specific requirements.
 	processEdge := func(fromNode *channeldb.LightningNode,
@@ -380,21 +382,26 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 			return
 		}
 
-		// If this vertex or edge has been black listed, then we'll
-		// skip exploring this edge.
-		if _, ok := ignoredNodes[fromVertex]; ok {
-			return
-		}
-
-		locator := newEdgeLocator(edge)
-		if _, ok := ignoredEdges[*locator]; ok {
-			return
-		}
-
+		// Calculate amount that the candidate node would have to sent
+		// out.
 		toNodeDist := distance[toNode]
-
 		amountToSend := toNodeDist.amountToReceive
 
+		// Request the success probability for this edge.
+		locator := newEdgeLocator(edge)
+		edgeProbability := r.ProbabilitySource(
+			fromVertex, *locator,
+		)
+
+		log.Tracef("path finding probability: fromnode=%v, chanid=%v, "+
+			"probability=%v", fromVertex, locator.ChannelID,
+			edgeProbability)
+
+		// If the probability is zero, there is no point in trying.
+		if edgeProbability == 0 {
+			return
+		}
+
 		// If the estimated bandwidth of the channel edge is not able
 		// to carry the amount that needs to be send, return.
 		if bandwidth < amountToSend {
@@ -453,19 +460,32 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 			return
 		}
 
+		// Calculate total probability of successfully reaching target
+		// by multiplying the probabilities. Both this edge and the rest
+		// of the route must succeed.
+		probability := toNodeDist.probability * edgeProbability
+
 		// By adding fromNode in the route, there will be an extra
 		// weight composed of the fee that this node will charge and
 		// the amount that will be locked for timeLockDelta blocks in
 		// the HTLC that is handed out to fromNode.
 		weight := edgeWeight(amountToReceive, fee, timeLockDelta)
 
-		// Compute the tentative distance to this new channel/edge
-		// which is the distance from our toNode to the target node
+		// Compute the tentative weight to this new channel/edge
+		// which is the weight from our toNode to the target node
 		// plus the weight of this edge.
-		tempDist := toNodeDist.dist + weight
+		tempWeight := toNodeDist.weight + weight
 
-		// If this new tentative distance is not better than the current
-		// best known distance to this node, return.
+		// Add an extra factor to the weight to take into account the
+		// probability.
+		tempDist := getProbabilityBasedDist(
+			tempWeight, probability, int64(r.PaymentAttemptPenalty),
+		)
+
+		// If the current best route is better than this candidate
+		// route, return. It is important to also return if the distance
+		// is equal, because otherwise the algorithm could run into an
+		// endless loop.
 		if tempDist >= distance[fromVertex].dist {
 			return
 		}
@@ -483,10 +503,11 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 		// map is populated with this edge.
 		distance[fromVertex] = nodeWithDist{
 			dist:            tempDist,
+			weight:          tempWeight,
 			node:            fromNode,
 			amountToReceive: amountToReceive,
-			fee:             fee,
 			incomingCltv:    incomingCltv,
+			probability:     probability,
 		}
 
 		next[fromVertex] = edge
@@ -614,5 +635,53 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
 			"too many hops")
 	}
 
+	log.Debugf("Found route: probability=%v, hops=%v, fee=%v\n",
+		distance[source].probability, numEdges,
+		distance[source].amountToReceive-amt)
+
 	return pathEdges, nil
 }
+
+// getProbabilityBasedDist converts a weight into a distance that takes into
+// account the success probability and the (virtual) cost of a failed payment
+// attempt.
+//
+// Derivation:
+//
+// Suppose there are two routes A and B with fees Fa and Fb and success
+// probabilities Pa and Pb.
+//
+// Is the expected cost of trying route A first and then B lower than trying the
+// other way around?
+//
+// The expected cost of A-then-B is: Pa*Fa + (1-Pa)*Pb*(c+Fb)
+//
+// The expected cost of B-then-A is: Pb*Fb + (1-Pb)*Pa*(c+Fa)
+//
+// In these equations, the term representing the case where both A and B fail is
+// left out because its value would be the same in both cases.
+//
+// Pa*Fa + (1-Pa)*Pb*(c+Fb) < Pb*Fb + (1-Pb)*Pa*(c+Fa)
+//
+// Pa*Fa + Pb*c + Pb*Fb - Pa*Pb*c - Pa*Pb*Fb < Pb*Fb + Pa*c + Pa*Fa - Pa*Pb*c - Pa*Pb*Fa
+//
+// Removing terms that cancel out:
+// Pb*c - Pa*Pb*Fb < Pa*c - Pa*Pb*Fa
+//
+// Divide by Pa*Pb:
+// c/Pa - Fb < c/Pb - Fa
+//
+// Move terms around:
+// Fa + c/Pa < Fb + c/Pb
+//
+// So the value of F + c/P can be used to compare routes.
+func getProbabilityBasedDist(weight int64, probability float64, penalty int64) int64 {
+	// Clamp probability to prevent overflow.
+	const minProbability = 0.00001
+
+	if probability < minProbability {
+		return infinity
+	}
+
+	return weight + int64(float64(penalty)/probability)
+}