Merge pull request #6463 from guggero/num-payments

ListPayments: add new optional count_total_payments flag

channeldb/payments.go

@@ -532,6 +532,10 @@ type PaymentsQuery struct {
 	// fully completed. This means that pending payments, as well as failed
 	// payments will show up if this field is set to true.
 	IncludeIncomplete bool
+
+	// CountTotal indicates that all payments currently present in the
+	// payment index (complete and incomplete) should be counted.
+	CountTotal bool
 }
 
 // PaymentsResponse contains the result of a query to the payments database.
@@ -555,6 +559,11 @@ type PaymentsResponse struct {
 	// in the event that the slice has too many events to fit into a single
 	// response. The offset can be used to continue forward pagination.
 	LastIndexOffset uint64
+
+	// TotalCount represents the total number of payments that are currently
+	// stored in the payment database. This will only be set if the
+	// CountTotal field in the query was set to true.
+	TotalCount uint64
 }
 
 // QueryPayments is a query to the payments database which is restricted
@@ -624,6 +633,35 @@ func (d *DB) QueryPayments(query PaymentsQuery) (PaymentsResponse, error) {
 			return err
 		}
 
+		// Counting the total number of payments is expensive, since we
+		// literally have to traverse the cursor linearly, which can
+		// take quite a while. So it's an optional query parameter.
+		if query.CountTotal {
+			var (
+				totalPayments uint64
+				err           error
+			)
+			countFn := func(_, _ []byte) error {
+				totalPayments++
+
+				return nil
+			}
+
+			// In non-boltdb database backends, there's a faster
+			// ForAll query that allows for batch fetching items.
+			if fastBucket, ok := indexes.(kvdb.ExtendedRBucket); ok {
+				err = fastBucket.ForAll(countFn)
+			} else {
+				err = indexes.ForEach(countFn)
+			}
+			if err != nil {
+				return fmt.Errorf("error counting payments: %v",
+					err)
+			}
+
+			resp.TotalCount = totalPayments
+		}
+
 		return nil
 	}, func() {
 		resp = PaymentsResponse{}

cmd/lncli/cmd_payments.go

@@ -1130,13 +1130,22 @@ var listPaymentsCommand = cli.Command{
 	Name:     "listpayments",
 	Category: "Payments",
 	Usage:    "List all outgoing payments.",
-	Description: "This command enables the retrieval of payments stored " +
+	Description: `
-		"in the database. Pagination is supported by the usage of " +
+	This command enables the retrieval of payments stored
-		"index_offset in combination with the paginate_forwards flag. " +
+	in the database.
-		"Reversed pagination is enabled by default to receive " +
+
-		"current payments first. Pagination can be resumed by using " +
+	Pagination is supported by the usage of index_offset in combination with
-		"the returned last_index_offset (for forwards order), or " +
+	the paginate_forwards flag.
-		"first_index_offset (for reversed order) as the offset_index. ",
+	Reversed pagination is enabled by default to receive current payments
+	first. Pagination can be resumed by using the returned last_index_offset
+	(for forwards order), or first_index_offset (for reversed order) as the
+	offset_index.
+
+	Because counting all payments in the payment database can take a long
+	time on systems with many payments, the count is not returned by
+	default. That feature can be turned on with the --count_total_payments
+	flag.
+	`,
 	Flags: []cli.Flag{
 		cli.BoolFlag{
 			Name: "include_incomplete",
@@ -1167,6 +1176,13 @@ var listPaymentsCommand = cli.Command{
 				"index_offset will be returned, allowing " +
 				"forwards pagination",
 		},
+		cli.BoolFlag{
+			Name: "count_total_payments",
+			Usage: "if set, all payments (complete or incomplete, " +
+				"independent of max_payments parameter) will " +
+				"be counted; can take a long time on systems " +
+				"with many payments",
+		},
 	},
 	Action: actionDecorator(listPayments),
 }
@@ -1177,10 +1193,11 @@ func listPayments(ctx *cli.Context) error {
 	defer cleanUp()
 
 	req := &lnrpc.ListPaymentsRequest{
-		IncludeIncomplete: ctx.Bool("include_incomplete"),
+		IncludeIncomplete:  ctx.Bool("include_incomplete"),
-		IndexOffset:       uint64(ctx.Uint("index_offset")),
+		IndexOffset:        uint64(ctx.Uint("index_offset")),
-		MaxPayments:       uint64(ctx.Uint("max_payments")),
+		MaxPayments:        uint64(ctx.Uint("max_payments")),
-		Reversed:          !ctx.Bool("paginate_forwards"),
+		Reversed:           !ctx.Bool("paginate_forwards"),
+		CountTotalPayments: ctx.Bool("count_total_payments"),
 	}
 
 	payments, err := client.ListPayments(ctxc, req)

docs/release-notes/release-notes-0.15.0.md

@@ -245,6 +245,11 @@ from occurring that would result in an erroneous force close.](https://github.co
 * [Expose](https://github.com/lightningnetwork/lnd/pull/6454) always on mode of
   the HTLC interceptor API through GetInfo.
 
+* [The `lnrpc.ListPayments` RPC now has an optional `count_total_payments`
+  parameter that will cause the `total_num_payments` response field to be set
+  to the total number of payments (complete and incomplete) that are currently
+  in the payment database](https://github.com/lightningnetwork/lnd/pull/6463).
+
 ## Database
 
 * [Add ForAll implementation for etcd to speed up

lnrpc/lightning.pb.go

@@ -12481,6 +12481,12 @@ type ListPaymentsRequest struct {
 	//specified index offset. This can be used to paginate backwards. The order
 	//of the returned payments is always oldest first (ascending index order).
 	Reversed bool `protobuf:"varint,4,opt,name=reversed,proto3" json:"reversed,omitempty"`
+	//
+	//If set, all payments (complete and incomplete, independent of the
+	//max_payments parameter) will be counted. Note that setting this to true will
+	//increase the run time of the call significantly on systems that have a lot
+	//of payments, as all of them have to be iterated through to be counted.
+	CountTotalPayments bool `protobuf:"varint,5,opt,name=count_total_payments,json=countTotalPayments,proto3" json:"count_total_payments,omitempty"`
 }
 
 func (x *ListPaymentsRequest) Reset() {
@@ -12543,6 +12549,13 @@ func (x *ListPaymentsRequest) GetReversed() bool {
 	return false
 }
 
+func (x *ListPaymentsRequest) GetCountTotalPayments() bool {
+	if x != nil {
+		return x.CountTotalPayments
+	}
+	return false
+}
+
 type ListPaymentsResponse struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -12558,6 +12571,12 @@ type ListPaymentsResponse struct {
 	//The index of the last item in the set of returned payments. This can be used
 	//as the index_offset to continue seeking forwards in the next request.
 	LastIndexOffset uint64 `protobuf:"varint,3,opt,name=last_index_offset,json=lastIndexOffset,proto3" json:"last_index_offset,omitempty"`
+	//
+	//Will only be set if count_total_payments in the request was set. Represents
+	//the total number of payments (complete and incomplete, independent of the
+	//number of payments requested in the query) currently present in the payments
+	//database.
+	TotalNumPayments uint64 `protobuf:"varint,4,opt,name=total_num_payments,json=totalNumPayments,proto3" json:"total_num_payments,omitempty"`
 }
 
 func (x *ListPaymentsResponse) Reset() {
@@ -12613,6 +12632,13 @@ func (x *ListPaymentsResponse) GetLastIndexOffset() uint64 {
 	return 0
 }
 
+func (x *ListPaymentsResponse) GetTotalNumPayments() uint64 {
+	if x != nil {
+		return x.TotalNumPayments
+	}
+	return 0
+}
+
 type DeletePaymentRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -18506,7 +18532,7 @@ var file_lightning_proto_rawDesc = []byte{
 	0x54, 0x4c, 0x43, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x4e, 0x5f,
 	0x46, 0x4c, 0x49, 0x47, 0x48, 0x54, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x55, 0x43, 0x43,
 	0x45, 0x45, 0x44, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45,
-	0x44, 0x10, 0x02, 0x22, 0xa6, 0x01, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6d,
+	0x44, 0x10, 0x02, 0x22, 0xd8, 0x01, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6d,
 	0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x12, 0x69,
 	0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x69, 0x6e, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74,
 	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65,
@@ -18516,17 +18542,23 @@ var file_lightning_proto_rawDesc = []byte{
 	0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20,
 	0x01, 0x28, 0x04, 0x52, 0x0b, 0x6d, 0x61, 0x78, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73,
 	0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01,
-	0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x65, 0x64, 0x22, 0x9c, 0x01, 0x0a,
+	0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x14,
-	0x14, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73,
+	0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x70, 0x61, 0x79, 0x6d,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x08, 0x70, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74,
+	0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x63, 0x6f, 0x75, 0x6e,
-	0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6c, 0x6e, 0x72, 0x70, 0x63, 0x2e,
+	0x74, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xca,
-	0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x70, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74,
+	0x01, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52,
-	0x73, 0x12, 0x2c, 0x0a, 0x12, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x08, 0x70, 0x61, 0x79, 0x6d, 0x65,
-	0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x66,
+	0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6c, 0x6e, 0x72, 0x70,
-	0x69, 0x72, 0x73, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12,
+	0x63, 0x2e, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x70, 0x61, 0x79, 0x6d, 0x65,
-	0x2a, 0x0a, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6f, 0x66,
+	0x6e, 0x74, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x69, 0x6e, 0x64,
-	0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x6c, 0x61, 0x73, 0x74,
+	0x65, 0x78, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52,
-	0x49, 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x65, 0x0a, 0x14, 0x44,
+	0x10, 0x66, 0x69, 0x72, 0x73, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x66, 0x66, 0x73, 0x65,
+	0x74, 0x12, 0x2a, 0x0a, 0x11, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f,
+	0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x6c, 0x61,
+	0x73, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x2c, 0x0a,
+	0x12, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x70, 0x61, 0x79, 0x6d, 0x65,
+	0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x74, 0x6f, 0x74, 0x61, 0x6c,
+	0x4e, 0x75, 0x6d, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x65, 0x0a, 0x14, 0x44,
 	0x65, 0x6c, 0x65, 0x74, 0x65, 0x50, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75,
 	0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x68,
 	0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x70, 0x61, 0x79, 0x6d, 0x65,

lnrpc/lightning.proto

@@ -3633,6 +3633,14 @@ message ListPaymentsRequest {
     of the returned payments is always oldest first (ascending index order).
     */
     bool reversed = 4;
+
+    /*
+    If set, all payments (complete and incomplete, independent of the
+    max_payments parameter) will be counted. Note that setting this to true will
+    increase the run time of the call significantly on systems that have a lot
+    of payments, as all of them have to be iterated through to be counted.
+    */
+    bool count_total_payments = 5;
 }
 
 message ListPaymentsResponse {
@@ -3650,6 +3658,14 @@ message ListPaymentsResponse {
     as the index_offset to continue seeking forwards in the next request.
     */
     uint64 last_index_offset = 3;
+
+    /*
+    Will only be set if count_total_payments in the request was set. Represents
+    the total number of payments (complete and incomplete, independent of the
+    number of payments requested in the query) currently present in the payments
+    database.
+    */
+    uint64 total_num_payments = 4;
 }
 
 message DeletePaymentRequest {

lnrpc/lightning.swagger.json

@@ -1916,6 +1916,13 @@
             "in": "query",
             "required": false,
             "type": "boolean"
+          },
+          {
+            "name": "count_total_payments",
+            "description": "If set, all payments (complete and incomplete, independent of the\nmax_payments parameter) will be counted. Note that setting this to true will\nincrease the run time of the call significantly on systems that have a lot\nof payments, as all of them have to be iterated through to be counted.",
+            "in": "query",
+            "required": false,
+            "type": "boolean"
           }
         ],
         "tags": [
@@ -5038,6 +5045,11 @@
           "type": "string",
           "format": "uint64",
           "description": "The index of the last item in the set of returned payments. This can be used\nas the index_offset to continue seeking forwards in the next request."
+        },
+        "total_num_payments": {
+          "type": "string",
+          "format": "uint64",
+          "description": "Will only be set if count_total_payments in the request was set. Represents\nthe total number of payments (complete and incomplete, independent of the\nnumber of payments requested in the query) currently present in the payments\ndatabase."
         }
       }
     },

rpcserver.go

@@ -6173,6 +6173,7 @@ func (r *rpcServer) ListPayments(ctx context.Context,
 		MaxPayments:       req.MaxPayments,
 		Reversed:          req.Reversed,
 		IncludeIncomplete: req.IncludeIncomplete,
+		CountTotal:        req.CountTotalPayments,
 	}
 
 	// If the maximum number of payments wasn't specified, then we'll
@@ -6189,6 +6190,7 @@ func (r *rpcServer) ListPayments(ctx context.Context,
 	paymentsResp := &lnrpc.ListPaymentsResponse{
 		LastIndexOffset:  paymentsQuerySlice.LastIndexOffset,
 		FirstIndexOffset: paymentsQuerySlice.FirstIndexOffset,
+		TotalNumPayments: paymentsQuerySlice.TotalCount,
 	}
 
 	for _, payment := range paymentsQuerySlice.Payments {