Merge pull request #10601 from ziggie1984/invoice-filter-index-optimization

invoices/sql_store: replace catch-all FilterInvoices with targeted index-friendly queries
This commit is contained in:
Olaoluwa Osuntokun authored and GitHub committed 2026-02-24 16:34:27 -08:00
commit 0b00c66231
7 files changed
+606 -146

No files matched your search

+2 -2
View File
@@ -113,8 +113,8 @@ func testInvoiceExpiryMigration(t *testing.T, makeDB makeMigrationTestDB) {
// AMP invoices.
err = migrate(TargetVersion(4))
invoices, err := db.FilterInvoices(ctxb, sqlc.FilterInvoicesParams{
AddIndexGet: SQLInt64(1),
invoices, err := db.FilterInvoicesByAddIndex(ctxb, sqlc.FilterInvoicesByAddIndexParams{
AddIndexGet: 1,
NumLimit: 100,
})
+275 -54
View File
@@ -64,74 +64,295 @@ func (q *Queries) DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) (s
)
}
const filterInvoices = `-- name: FilterInvoices :many
const fetchPendingInvoices = `-- name: FetchPendingInvoices :many
SELECT
invoices.id, invoices.hash, invoices.preimage, invoices.settle_index, invoices.settled_at, invoices.memo, invoices.amount_msat, invoices.cltv_delta, invoices.expiry, invoices.payment_addr, invoices.payment_request, invoices.payment_request_hash, invoices.state, invoices.amount_paid_msat, invoices.is_amp, invoices.is_hodl, invoices.is_keysend, invoices.created_at
FROM invoices
WHERE (
id >= $1 OR
$1 IS NULL
) AND (
id <= $2 OR
$2 IS NULL
) AND (
settle_index >= $3 OR
$3 IS NULL
) AND (
settle_index <= $4 OR
$4 IS NULL
) AND (
state = $5 OR
$5 IS NULL
) AND (
created_at >= $6 OR
$6 IS NULL
) AND (
created_at < $7 OR
$7 IS NULL
) AND (
CASE
WHEN $8 = TRUE THEN (state = 0 OR state = 3)
ELSE TRUE
END
)
ORDER BY
CASE
WHEN $9 = FALSE OR $9 IS NULL THEN id
ELSE NULL
END ASC,
CASE
WHEN $9 = TRUE THEN id
ELSE NULL
END DESC
LIMIT $11 OFFSET $10
WHERE state IN (0, 3) -- 0 = ContractOpen, 3 = ContractAccepted
ORDER BY id ASC
LIMIT $2 OFFSET $1
`
type FilterInvoicesParams struct {
AddIndexGet sql.NullInt64
AddIndexLet sql.NullInt64
type FetchPendingInvoicesParams struct {
NumOffset int32
NumLimit int32
}
// FetchPendingInvoices returns all invoices in a pending state (open or
// accepted). The invoices_state_idx index on the state column makes this a
// fast index scan rather than a full table scan.
func (q *Queries) FetchPendingInvoices(ctx context.Context, arg FetchPendingInvoicesParams) ([]Invoice, error) {
rows, err := q.db.QueryContext(ctx, fetchPendingInvoices, arg.NumOffset, arg.NumLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Invoice
for rows.Next() {
var i Invoice
if err := rows.Scan(
&i.ID,
&i.Hash,
&i.Preimage,
&i.SettleIndex,
&i.SettledAt,
&i.Memo,
&i.AmountMsat,
&i.CltvDelta,
&i.Expiry,
&i.PaymentAddr,
&i.PaymentRequest,
&i.PaymentRequestHash,
&i.State,
&i.AmountPaidMsat,
&i.IsAmp,
&i.IsHodl,
&i.IsKeysend,
&i.CreatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const filterInvoicesByAddIndex = `-- name: FilterInvoicesByAddIndex :many
SELECT
invoices.id, invoices.hash, invoices.preimage, invoices.settle_index, invoices.settled_at, invoices.memo, invoices.amount_msat, invoices.cltv_delta, invoices.expiry, invoices.payment_addr, invoices.payment_request, invoices.payment_request_hash, invoices.state, invoices.amount_paid_msat, invoices.is_amp, invoices.is_hodl, invoices.is_keysend, invoices.created_at
FROM invoices
WHERE id >= $1
ORDER BY id ASC
LIMIT $3 OFFSET $2
`
type FilterInvoicesByAddIndexParams struct {
AddIndexGet int64
NumOffset int32
NumLimit int32
}
// FilterInvoicesByAddIndex returns invoices whose add_index (primary key id)
// is greater than or equal to the given value, ordered by id. Because id is
// the primary key, this is always an efficient range scan on the clustered
// index.
func (q *Queries) FilterInvoicesByAddIndex(ctx context.Context, arg FilterInvoicesByAddIndexParams) ([]Invoice, error) {
rows, err := q.db.QueryContext(ctx, filterInvoicesByAddIndex, arg.AddIndexGet, arg.NumOffset, arg.NumLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Invoice
for rows.Next() {
var i Invoice
if err := rows.Scan(
&i.ID,
&i.Hash,
&i.Preimage,
&i.SettleIndex,
&i.SettledAt,
&i.Memo,
&i.AmountMsat,
&i.CltvDelta,
&i.Expiry,
&i.PaymentAddr,
&i.PaymentRequest,
&i.PaymentRequestHash,
&i.State,
&i.AmountPaidMsat,
&i.IsAmp,
&i.IsHodl,
&i.IsKeysend,
&i.CreatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const filterInvoicesBySettleIndex = `-- name: FilterInvoicesBySettleIndex :many
SELECT
invoices.id, invoices.hash, invoices.preimage, invoices.settle_index, invoices.settled_at, invoices.memo, invoices.amount_msat, invoices.cltv_delta, invoices.expiry, invoices.payment_addr, invoices.payment_request, invoices.payment_request_hash, invoices.state, invoices.amount_paid_msat, invoices.is_amp, invoices.is_hodl, invoices.is_keysend, invoices.created_at
FROM invoices
WHERE settle_index >= $1
ORDER BY id ASC
LIMIT $3 OFFSET $2
`
type FilterInvoicesBySettleIndexParams struct {
SettleIndexGet sql.NullInt64
SettleIndexLet sql.NullInt64
State sql.NullInt16
CreatedAfter sql.NullTime
CreatedBefore sql.NullTime
PendingOnly interface{}
Reverse interface{}
NumOffset int32
NumLimit int32
}
func (q *Queries) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) ([]Invoice, error) {
rows, err := q.db.QueryContext(ctx, filterInvoices,
// FilterInvoicesBySettleIndex returns settled invoices whose settle_index is
// greater than or equal to the given value, ordered by id. The caller must
// always supply a concrete lower bound so the invoices_settle_index_idx index
// can be used.
func (q *Queries) FilterInvoicesBySettleIndex(ctx context.Context, arg FilterInvoicesBySettleIndexParams) ([]Invoice, error) {
rows, err := q.db.QueryContext(ctx, filterInvoicesBySettleIndex, arg.SettleIndexGet, arg.NumOffset, arg.NumLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Invoice
for rows.Next() {
var i Invoice
if err := rows.Scan(
&i.ID,
&i.Hash,
&i.Preimage,
&i.SettleIndex,
&i.SettledAt,
&i.Memo,
&i.AmountMsat,
&i.CltvDelta,
&i.Expiry,
&i.PaymentAddr,
&i.PaymentRequest,
&i.PaymentRequestHash,
&i.State,
&i.AmountPaidMsat,
&i.IsAmp,
&i.IsHodl,
&i.IsKeysend,
&i.CreatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const filterInvoicesForward = `-- name: FilterInvoicesForward :many
SELECT
invoices.id, invoices.hash, invoices.preimage, invoices.settle_index, invoices.settled_at, invoices.memo, invoices.amount_msat, invoices.cltv_delta, invoices.expiry, invoices.payment_addr, invoices.payment_request, invoices.payment_request_hash, invoices.state, invoices.amount_paid_msat, invoices.is_amp, invoices.is_hodl, invoices.is_keysend, invoices.created_at
FROM invoices
WHERE id >= $1
AND (NOT $2 OR state IN (0, 3)) -- 0 = ContractOpen, 3 = ContractAccepted
AND created_at >= $3
AND created_at < $4
ORDER BY id ASC
LIMIT $6 OFFSET $5
`
type FilterInvoicesForwardParams struct {
AddIndexGet int64
PendingOnly interface{}
CreatedAfter time.Time
CreatedBefore time.Time
NumOffset int32
NumLimit int32
}
// FilterInvoicesForward returns invoices in ascending id order starting from
// add_index_get. All parameters are non-nullable so the planner always sees
// plain range predicates and can use the primary-key index. The caller is
// responsible for supplying Go-side defaults when a filter is not needed:
//
// created_after → time.Unix(0, 0).UTC() (epoch – before any invoice)
// created_before → time.Date(9999, …) (far future – no upper cap)
// pending_only → false (include all states)
func (q *Queries) FilterInvoicesForward(ctx context.Context, arg FilterInvoicesForwardParams) ([]Invoice, error) {
rows, err := q.db.QueryContext(ctx, filterInvoicesForward,
arg.AddIndexGet,
arg.AddIndexLet,
arg.SettleIndexGet,
arg.SettleIndexLet,
arg.State,
arg.PendingOnly,
arg.CreatedAfter,
arg.CreatedBefore,
arg.NumOffset,
arg.NumLimit,
)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Invoice
for rows.Next() {
var i Invoice
if err := rows.Scan(
&i.ID,
&i.Hash,
&i.Preimage,
&i.SettleIndex,
&i.SettledAt,
&i.Memo,
&i.AmountMsat,
&i.CltvDelta,
&i.Expiry,
&i.PaymentAddr,
&i.PaymentRequest,
&i.PaymentRequestHash,
&i.State,
&i.AmountPaidMsat,
&i.IsAmp,
&i.IsHodl,
&i.IsKeysend,
&i.CreatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const filterInvoicesReverse = `-- name: FilterInvoicesReverse :many
SELECT
invoices.id, invoices.hash, invoices.preimage, invoices.settle_index, invoices.settled_at, invoices.memo, invoices.amount_msat, invoices.cltv_delta, invoices.expiry, invoices.payment_addr, invoices.payment_request, invoices.payment_request_hash, invoices.state, invoices.amount_paid_msat, invoices.is_amp, invoices.is_hodl, invoices.is_keysend, invoices.created_at
FROM invoices
WHERE id <= $1
AND (NOT $2 OR state IN (0, 3)) -- 0 = ContractOpen, 3 = ContractAccepted
AND created_at >= $3
AND created_at < $4
ORDER BY id DESC
LIMIT $6 OFFSET $5
`
type FilterInvoicesReverseParams struct {
AddIndexLet int64
PendingOnly interface{}
CreatedAfter time.Time
CreatedBefore time.Time
NumOffset int32
NumLimit int32
}
// FilterInvoicesReverse is the descending counterpart of FilterInvoicesForward.
// It returns invoices in descending id order up to and including add_index_let.
// See FilterInvoicesForward for the expected Go-side defaults.
func (q *Queries) FilterInvoicesReverse(ctx context.Context, arg FilterInvoicesReverseParams) ([]Invoice, error) {
rows, err := q.db.QueryContext(ctx, filterInvoicesReverse,
arg.AddIndexLet,
arg.PendingOnly,
arg.Reverse,
arg.CreatedAfter,
arg.CreatedBefore,
arg.NumOffset,
arg.NumLimit,
)
+26 -1
View File
@@ -31,8 +31,33 @@ type Querier interface {
DeleteZombieChannel(ctx context.Context, arg DeleteZombieChannelParams) (sql.Result, error)
FetchAMPSubInvoiceHTLCs(ctx context.Context, arg FetchAMPSubInvoiceHTLCsParams) ([]FetchAMPSubInvoiceHTLCsRow, error)
FetchAMPSubInvoices(ctx context.Context, arg FetchAMPSubInvoicesParams) ([]AmpSubInvoice, error)
// FetchPendingInvoices returns all invoices in a pending state (open or
// accepted). The invoices_state_idx index on the state column makes this a
// fast index scan rather than a full table scan.
FetchPendingInvoices(ctx context.Context, arg FetchPendingInvoicesParams) ([]Invoice, error)
FetchSettledAMPSubInvoices(ctx context.Context, arg FetchSettledAMPSubInvoicesParams) ([]FetchSettledAMPSubInvoicesRow, error)
FilterInvoices(ctx context.Context, arg FilterInvoicesParams) ([]Invoice, error)
// FilterInvoicesByAddIndex returns invoices whose add_index (primary key id)
// is greater than or equal to the given value, ordered by id. Because id is
// the primary key, this is always an efficient range scan on the clustered
// index.
FilterInvoicesByAddIndex(ctx context.Context, arg FilterInvoicesByAddIndexParams) ([]Invoice, error)
// FilterInvoicesBySettleIndex returns settled invoices whose settle_index is
// greater than or equal to the given value, ordered by id. The caller must
// always supply a concrete lower bound so the invoices_settle_index_idx index
// can be used.
FilterInvoicesBySettleIndex(ctx context.Context, arg FilterInvoicesBySettleIndexParams) ([]Invoice, error)
// FilterInvoicesForward returns invoices in ascending id order starting from
// add_index_get. All parameters are non-nullable so the planner always sees
// plain range predicates and can use the primary-key index. The caller is
// responsible for supplying Go-side defaults when a filter is not needed:
// created_after → time.Unix(0, 0).UTC() (epoch – before any invoice)
// created_before → time.Date(9999, …) (far future – no upper cap)
// pending_only → false (include all states)
FilterInvoicesForward(ctx context.Context, arg FilterInvoicesForwardParams) ([]Invoice, error)
// FilterInvoicesReverse is the descending counterpart of FilterInvoicesForward.
// It returns invoices in descending id order up to and including add_index_let.
// See FilterInvoicesForward for the expected Go-side defaults.
FilterInvoicesReverse(ctx context.Context, arg FilterInvoicesReverseParams) ([]Invoice, error)
GetAMPInvoiceID(ctx context.Context, setID []byte) (int64, error)
GetChannelAndNodesBySCID(ctx context.Context, arg GetChannelAndNodesBySCIDParams) (GetChannelAndNodesBySCIDRow, error)
GetChannelByOutpointWithPolicies(ctx context.Context, arg GetChannelByOutpointWithPoliciesParams) (GetChannelByOutpointWithPoliciesRow, error)
+62 -37
View File
@@ -65,46 +65,71 @@ FROM invoices i
INNER JOIN amp_sub_invoices a
ON i.id = a.invoice_id AND a.set_id = $1;
-- name: FilterInvoices :many
-- name: FetchPendingInvoices :many
-- FetchPendingInvoices returns all invoices in a pending state (open or
-- accepted). The invoices_state_idx index on the state column makes this a
-- fast index scan rather than a full table scan.
SELECT
invoices.*
FROM invoices
WHERE (
id >= sqlc.narg('add_index_get') OR
sqlc.narg('add_index_get') IS NULL
) AND (
id <= sqlc.narg('add_index_let') OR
sqlc.narg('add_index_let') IS NULL
) AND (
settle_index >= sqlc.narg('settle_index_get') OR
sqlc.narg('settle_index_get') IS NULL
) AND (
settle_index <= sqlc.narg('settle_index_let') OR
sqlc.narg('settle_index_let') IS NULL
) AND (
state = sqlc.narg('state') OR
sqlc.narg('state') IS NULL
) AND (
created_at >= sqlc.narg('created_after') OR
sqlc.narg('created_after') IS NULL
) AND (
created_at < sqlc.narg('created_before') OR
sqlc.narg('created_before') IS NULL
) AND (
CASE
WHEN sqlc.narg('pending_only') = TRUE THEN (state = 0 OR state = 3)
ELSE TRUE
END
)
ORDER BY
CASE
WHEN sqlc.narg('reverse') = FALSE OR sqlc.narg('reverse') IS NULL THEN id
ELSE NULL
END ASC,
CASE
WHEN sqlc.narg('reverse') = TRUE THEN id
ELSE NULL
END DESC
WHERE state IN (0, 3) -- 0 = ContractOpen, 3 = ContractAccepted
ORDER BY id ASC
LIMIT @num_limit OFFSET @num_offset;
-- name: FilterInvoicesBySettleIndex :many
-- FilterInvoicesBySettleIndex returns settled invoices whose settle_index is
-- greater than or equal to the given value, ordered by id. The caller must
-- always supply a concrete lower bound so the invoices_settle_index_idx index
-- can be used.
SELECT
invoices.*
FROM invoices
WHERE settle_index >= @settle_index_get
ORDER BY id ASC
LIMIT @num_limit OFFSET @num_offset;
-- name: FilterInvoicesByAddIndex :many
-- FilterInvoicesByAddIndex returns invoices whose add_index (primary key id)
-- is greater than or equal to the given value, ordered by id. Because id is
-- the primary key, this is always an efficient range scan on the clustered
-- index.
SELECT
invoices.*
FROM invoices
WHERE id >= @add_index_get
ORDER BY id ASC
LIMIT @num_limit OFFSET @num_offset;
-- name: FilterInvoicesForward :many
-- FilterInvoicesForward returns invoices in ascending id order starting from
-- add_index_get. All parameters are non-nullable so the planner always sees
-- plain range predicates and can use the primary-key index. The caller is
-- responsible for supplying Go-side defaults when a filter is not needed:
-- created_after → time.Unix(0, 0).UTC() (epoch – before any invoice)
-- created_before → time.Date(9999, …) (far future – no upper cap)
-- pending_only → false (include all states)
SELECT
invoices.*
FROM invoices
WHERE id >= @add_index_get
AND (NOT @pending_only OR state IN (0, 3)) -- 0 = ContractOpen, 3 = ContractAccepted
AND created_at >= @created_after
AND created_at < @created_before
ORDER BY id ASC
LIMIT @num_limit OFFSET @num_offset;
-- name: FilterInvoicesReverse :many
-- FilterInvoicesReverse is the descending counterpart of FilterInvoicesForward.
-- It returns invoices in descending id order up to and including add_index_let.
-- See FilterInvoicesForward for the expected Go-side defaults.
SELECT
invoices.*
FROM invoices
WHERE id <= @add_index_let
AND (NOT @pending_only OR state IN (0, 3)) -- 0 = ContractOpen, 3 = ContractAccepted
AND created_at >= @created_after
AND created_at < @created_before
ORDER BY id DESC
LIMIT @num_limit OFFSET @num_offset;
-- name: UpdateInvoiceState :execresult