Merge pull request #10619 from ziggie1984/increase-settling-speed

invoices/sql: fix full table scans on HTLC settlement hot path

sqldb/migrations_dev.go

@@ -22,4 +22,9 @@ var migrationAdditions = []MigrationConfig{
 		// schema. This is optional and can be disabled by the
 		// user if necessary.
 	},
+	{
+		Name:          "000012_drop_redundant_invoice_indexes",
+		Version:       15,
+		SchemaVersion: 12,
+	},
 }

sqldb/sqlc/invoices.sql.go

@@ -396,84 +396,36 @@ func (q *Queries) FilterInvoicesReverse(ctx context.Context, arg FilterInvoicesR
 	return items, nil
 }
 
-const getInvoice = `-- name: GetInvoice :many
-
+const getInvoiceByAddr = `-- name: GetInvoiceByAddr :one
 SELECT i.id, i.hash, i.preimage, i.settle_index, i.settled_at, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.payment_request_hash, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at
 FROM invoices i
-LEFT JOIN amp_sub_invoices a 
-ON i.id = a.invoice_id
-AND (
-    a.set_id = $1 OR $1 IS NULL
-)
-WHERE (
-    i.id = $2 OR 
-    $2 IS NULL
-) AND (
-    i.hash = $3 OR 
-    $3 IS NULL
-) AND (
-    i.payment_addr = $4 OR 
-    $4 IS NULL
-)
-GROUP BY i.id
-LIMIT 2
+WHERE i.payment_addr = $1
 `
 
-type GetInvoiceParams struct {
-	SetID       []byte
-	AddIndex    sql.NullInt64
-	Hash        []byte
-	PaymentAddr []byte
-}
-
-// This method may return more than one invoice if filter using multiple fields
-// from different invoices. It is the caller's responsibility to ensure that
-// we bubble up an error in those cases.
-func (q *Queries) GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoice, error) {
-	rows, err := q.db.QueryContext(ctx, getInvoice,
-		arg.SetID,
-		arg.AddIndex,
-		arg.Hash,
-		arg.PaymentAddr,
+func (q *Queries) GetInvoiceByAddr(ctx context.Context, paymentAddr []byte) (Invoice, error) {
+	row := q.db.QueryRowContext(ctx, getInvoiceByAddr, paymentAddr)
+	var i Invoice
+	err := row.Scan(
+		&i.ID,
+		&i.Hash,
+		&i.Preimage,
+		&i.SettleIndex,
+		&i.SettledAt,
+		&i.Memo,
+		&i.AmountMsat,
+		&i.CltvDelta,
+		&i.Expiry,
+		&i.PaymentAddr,
+		&i.PaymentRequest,
+		&i.PaymentRequestHash,
+		&i.State,
+		&i.AmountPaidMsat,
+		&i.IsAmp,
+		&i.IsHodl,
+		&i.IsKeysend,
+		&i.CreatedAt,
 	)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []Invoice
-	for rows.Next() {
-		var i Invoice
-		if err := rows.Scan(
-			&i.ID,
-			&i.Hash,
-			&i.Preimage,
-			&i.SettleIndex,
-			&i.SettledAt,
-			&i.Memo,
-			&i.AmountMsat,
-			&i.CltvDelta,
-			&i.Expiry,
-			&i.PaymentAddr,
-			&i.PaymentRequest,
-			&i.PaymentRequestHash,
-			&i.State,
-			&i.AmountPaidMsat,
-			&i.IsAmp,
-			&i.IsHodl,
-			&i.IsKeysend,
-			&i.CreatedAt,
-		); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Close(); err != nil {
-		return nil, err
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
+	return i, err
 }
 
 const getInvoiceByHash = `-- name: GetInvoiceByHash :one
@@ -515,6 +467,8 @@ INNER JOIN amp_sub_invoices a
 ON i.id = a.invoice_id AND a.set_id = $1
 `
 
+// TODO(ziggie): This query can only return one invoice if the set_id is
+// the primary key of amp_sub_invoices table.
 func (q *Queries) GetInvoiceBySetID(ctx context.Context, setID []byte) ([]Invoice, error) {
 	rows, err := q.db.QueryContext(ctx, getInvoiceBySetID, setID)
 	if err != nil {

sqldb/sqlc/migrations/000012_drop_redundant_invoice_indexes.down.sql

@@ -0,0 +1,4 @@
+CREATE INDEX IF NOT EXISTS invoices_hash_idx ON invoices(hash);
+CREATE INDEX IF NOT EXISTS invoices_payment_addr_idx ON invoices(payment_addr);
+CREATE INDEX IF NOT EXISTS invoices_preimage_idx ON invoices(preimage);
+CREATE INDEX IF NOT EXISTS invoices_settled_at_idx ON invoices(settled_at);

sqldb/sqlc/migrations/000012_drop_redundant_invoice_indexes.up.sql

@@ -0,0 +1,16 @@
+-- invoices_hash_idx is redundant: The UNIQUE constraint on invoices(hash)
+-- already creates an implicit index.
+DROP INDEX IF EXISTS invoices_hash_idx;
+
+-- invoices_payment_addr_idx is redundant: The UNIQUE constraint on
+-- invoices(payment_addr) already creates an implicit index.
+DROP INDEX IF EXISTS invoices_payment_addr_idx;
+
+-- invoices_preimage_idx is useless: There are no queries that filter on 
+-- preimage so there is no need to index it.
+DROP INDEX IF EXISTS invoices_preimage_idx;
+
+-- invoices_settled_at_idx is useless: settled_at is NULL for all pending
+-- invoices and is never used as a filter in any query (settle_index is used
+-- instead).
+DROP INDEX IF EXISTS invoices_settled_at_idx;

sqldb/sqlc/querier.go

@@ -107,11 +107,10 @@ type Querier interface {
 	GetClosedChannelsSCIDs(ctx context.Context, scids [][]byte) ([][]byte, error)
 	GetDatabaseVersion(ctx context.Context) (int32, error)
 	GetExtraNodeTypes(ctx context.Context, nodeID int64) ([]GraphNodeExtraType, error)
-	// This method may return more than one invoice if filter using multiple fields
-	// from different invoices. It is the caller's responsibility to ensure that
-	// we bubble up an error in those cases.
-	GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoice, error)
+	GetInvoiceByAddr(ctx context.Context, paymentAddr []byte) (Invoice, error)
 	GetInvoiceByHash(ctx context.Context, hash []byte) (Invoice, error)
+	// TODO(ziggie): This query can only return one invoice if the set_id is
+	// the primary key of amp_sub_invoices table.
 	GetInvoiceBySetID(ctx context.Context, setID []byte) ([]Invoice, error)
 	GetInvoiceFeatures(ctx context.Context, invoiceID int64) ([]InvoiceFeature, error)
 	GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error)

sqldb/sqlc/queries/invoices.sql

@@ -29,37 +29,19 @@ SELECT *
 FROM invoice_features
 WHERE invoice_id = $1;
 
--- This method may return more than one invoice if filter using multiple fields
--- from different invoices. It is the caller's responsibility to ensure that 
--- we bubble up an error in those cases.
-
--- name: GetInvoice :many
-SELECT i.*
-FROM invoices i
-LEFT JOIN amp_sub_invoices a 
-ON i.id = a.invoice_id
-AND (
-    a.set_id = sqlc.narg('set_id') OR sqlc.narg('set_id') IS NULL
-)
-WHERE (
-    i.id = sqlc.narg('add_index') OR 
-    sqlc.narg('add_index') IS NULL
-) AND (
-    i.hash = sqlc.narg('hash') OR 
-    sqlc.narg('hash') IS NULL
-) AND (
-    i.payment_addr = sqlc.narg('payment_addr') OR 
-    sqlc.narg('payment_addr') IS NULL
-)
-GROUP BY i.id
-LIMIT 2;
-
 -- name: GetInvoiceByHash :one
 SELECT i.*
 FROM invoices i
 WHERE i.hash = $1;
 
+-- name: GetInvoiceByAddr :one
+SELECT i.*
+FROM invoices i
+WHERE i.payment_addr = $1;
+
 -- name: GetInvoiceBySetID :many
+-- TODO(ziggie): This query can only return one invoice if the set_id is 
+-- the primary key of amp_sub_invoices table.
 SELECT i.*
 FROM invoices i
 INNER JOIN amp_sub_invoices a