From e7d82fa530087586475821ab50485393058fd388 Mon Sep 17 00:00:00 2001 From: positiveblue Date: Fri, 22 Sep 2023 12:30:15 -0700 Subject: [PATCH 01/21] tools: update sqlc version to v1.25.0 --- scripts/gen_sqlc_docker.sh | 2 +- sqldb/sqlc/amp_invoices.sql.go | 2 +- sqldb/sqlc/db.go | 2 +- sqldb/sqlc/invoice_events.sql.go | 2 +- sqldb/sqlc/invoices.sql.go | 2 +- sqldb/sqlc/models.go | 2 +- sqldb/sqlc/querier.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/gen_sqlc_docker.sh b/scripts/gen_sqlc_docker.sh index 9c8e88487..e011d689e 100755 --- a/scripts/gen_sqlc_docker.sh +++ b/scripts/gen_sqlc_docker.sh @@ -16,4 +16,4 @@ docker run \ -e UID=$UID \ -v "$DIR/../:/build" \ -w /build \ - kjconroy/sqlc:1.18.0 generate + sqlc/sqlc:1.25.0 generate diff --git a/sqldb/sqlc/amp_invoices.sql.go b/sqldb/sqlc/amp_invoices.sql.go index c803a8307..46c368630 100644 --- a/sqldb/sqlc/amp_invoices.sql.go +++ b/sqldb/sqlc/amp_invoices.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 // source: amp_invoices.sql package sqlc diff --git a/sqldb/sqlc/db.go b/sqldb/sqlc/db.go index 9848f3e6a..8ed64d139 100644 --- a/sqldb/sqlc/db.go +++ b/sqldb/sqlc/db.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 package sqlc diff --git a/sqldb/sqlc/invoice_events.sql.go b/sqldb/sqlc/invoice_events.sql.go index 054d9091c..b0623fc1e 100644 --- a/sqldb/sqlc/invoice_events.sql.go +++ b/sqldb/sqlc/invoice_events.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 // source: invoice_events.sql package sqlc diff --git a/sqldb/sqlc/invoices.sql.go b/sqldb/sqlc/invoices.sql.go index fa8d9c294..927905c96 100644 --- a/sqldb/sqlc/invoices.sql.go +++ b/sqldb/sqlc/invoices.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 // source: invoices.sql package sqlc diff --git a/sqldb/sqlc/models.go b/sqldb/sqlc/models.go index b0ef3c857..9a72ae2ce 100644 --- a/sqldb/sqlc/models.go +++ b/sqldb/sqlc/models.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 package sqlc diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 1e766ac63..b879f06d3 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 package sqlc From 3e0f98a75af4e7e38d646a37e7174b55cd5f59b3 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 22 Nov 2023 17:23:01 +0100 Subject: [PATCH 02/21] sqldb: introduce replace for the BIGINT type alias --- sqldb/postgres.go | 3 ++- sqldb/sqlite.go | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/sqldb/postgres.go b/sqldb/postgres.go index 3cfa59efc..9aac2a489 100644 --- a/sqldb/postgres.go +++ b/sqldb/postgres.go @@ -89,7 +89,7 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { // system. // // First, we'll need to open up a new migration instance for - // our current target database: sqlite. + // our current target database: Postgres. driver, err := postgres_migrate.WithInstance( rawDB, &postgres_migrate.Config{}, ) @@ -100,6 +100,7 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { postgresFS := newReplacerFS(sqlSchemas, map[string]string{ "BLOB": "BYTEA", "INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY", + "BIGINT PRIMARY KEY": "BIGSERIAL PRIMARY KEY", "TIMESTAMP": "TIMESTAMP WITHOUT TIME ZONE", }) diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index 8ecadd68d..4b9895999 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -133,8 +133,16 @@ func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) { return nil, err } + // We use INTEGER PRIMARY KEY for sqlite, because it acts as a + // ROWID alias which is 8 bytes big and also autoincrements. + // It's important to use the ROWID as a primary key because the + // key look ups are almost twice as fast + sqliteFS := newReplacerFS(sqlSchemas, map[string]string{ + "BIGINT PRIMARY KEY": "INTEGER PRIMARY KEY", + }) + err = applyMigrations( - sqlSchemas, driver, "sqlc/migrations", "sqlc", + sqliteFS, driver, "sqlc/migrations", "sqlc", ) if err != nil { return nil, err From 6a360fb2e247bd3812538b3d0db4381f5769c99d Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 22 Nov 2023 17:23:23 +0100 Subject: [PATCH 03/21] sqldb: simplify and fixup the existing invoice store schema and queries This commit attempts to fix some issues with the invoice store's schema that we couldn't foresee before the implementation was finished. This is safe as the schema has not been instantiated yet outside of unit tests. Furthermore the commit updates invoice store SQL queries according to fixes in the schema as well as to prepare the higher level implementation in the upcoming commits. --- sqldb/sqlc/amp_invoices.sql.go | 418 +++++++++-------- sqldb/sqlc/invoice_events.sql.go | 196 ++++---- sqldb/sqlc/invoices.sql.go | 442 +++++++----------- sqldb/sqlc/migrations/000001_invoices.up.sql | 62 +-- .../migrations/000002_amp_invoices.up.sql | 42 +- .../migrations/000003_invoice_events.down.sql | 9 +- .../migrations/000003_invoice_events.up.sql | 90 ++-- sqldb/sqlc/models.go | 82 ++-- sqldb/sqlc/querier.go | 53 +-- sqldb/sqlc/queries/amp_invoices.sql | 124 +++-- sqldb/sqlc/queries/invoice_events.sql | 66 +-- sqldb/sqlc/queries/invoices.sql | 134 ++---- 12 files changed, 781 insertions(+), 937 deletions(-) diff --git a/sqldb/sqlc/amp_invoices.sql.go b/sqldb/sqlc/amp_invoices.sql.go index 46c368630..3fcfe4b27 100644 --- a/sqldb/sqlc/amp_invoices.sql.go +++ b/sqldb/sqlc/amp_invoices.sql.go @@ -11,67 +11,69 @@ import ( "time" ) -const deleteAMPHTLCCustomRecords = `-- name: DeleteAMPHTLCCustomRecords :exec -WITH htlc_ids AS ( - SELECT htlc_id - FROM amp_invoice_htlcs - WHERE invoice_id = $1 +const fetchAMPSubInvoiceHTLCs = `-- name: FetchAMPSubInvoiceHTLCs :many +SELECT + amp.set_id, amp.root_share, amp.child_index, amp.hash, amp.preimage, + invoice_htlcs.id, invoice_htlcs.chan_id, invoice_htlcs.htlc_id, invoice_htlcs.amount_msat, invoice_htlcs.total_mpp_msat, invoice_htlcs.accept_height, invoice_htlcs.accept_time, invoice_htlcs.expiry_height, invoice_htlcs.state, invoice_htlcs.resolve_time, invoice_htlcs.invoice_id +FROM amp_sub_invoice_htlcs amp +INNER JOIN invoice_htlcs ON amp.htlc_id = invoice_htlcs.id +WHERE amp.invoice_id = $1 +AND ( + set_id = $2 OR + $2 IS NULL ) -DELETE -FROM invoice_htlc_custom_records -WHERE htlc_id IN (SELECT id FROM htlc_ids) ` -func (q *Queries) DeleteAMPHTLCCustomRecords(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteAMPHTLCCustomRecords, invoiceID) - return err +type FetchAMPSubInvoiceHTLCsParams struct { + InvoiceID int64 + SetID []byte } -const deleteAMPHTLCs = `-- name: DeleteAMPHTLCs :exec -DELETE -FROM amp_invoice_htlcs -WHERE invoice_id = $1 -` - -func (q *Queries) DeleteAMPHTLCs(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteAMPHTLCs, invoiceID) - return err +type FetchAMPSubInvoiceHTLCsRow struct { + SetID []byte + RootShare []byte + ChildIndex int64 + Hash []byte + Preimage []byte + ID int64 + ChanID string + HtlcID int64 + AmountMsat int64 + TotalMppMsat sql.NullInt64 + AcceptHeight int32 + AcceptTime time.Time + ExpiryHeight int32 + State int16 + ResolveTime sql.NullTime + InvoiceID int64 } -const deleteAMPInvoiceHTLC = `-- name: DeleteAMPInvoiceHTLC :exec -DELETE -FROM amp_invoice_htlcs -WHERE set_id = $1 -` - -func (q *Queries) DeleteAMPInvoiceHTLC(ctx context.Context, setID []byte) error { - _, err := q.db.ExecContext(ctx, deleteAMPInvoiceHTLC, setID) - return err -} - -const getAMPInvoiceHTLCsByInvoiceID = `-- name: GetAMPInvoiceHTLCsByInvoiceID :many -SELECT set_id, htlc_id, invoice_id, root_share, child_index, hash, preimage -FROM amp_invoice_htlcs -WHERE invoice_id = $1 -` - -func (q *Queries) GetAMPInvoiceHTLCsByInvoiceID(ctx context.Context, invoiceID int32) ([]AmpInvoiceHtlc, error) { - rows, err := q.db.QueryContext(ctx, getAMPInvoiceHTLCsByInvoiceID, invoiceID) +func (q *Queries) FetchAMPSubInvoiceHTLCs(ctx context.Context, arg FetchAMPSubInvoiceHTLCsParams) ([]FetchAMPSubInvoiceHTLCsRow, error) { + rows, err := q.db.QueryContext(ctx, fetchAMPSubInvoiceHTLCs, arg.InvoiceID, arg.SetID) if err != nil { return nil, err } defer rows.Close() - var items []AmpInvoiceHtlc + var items []FetchAMPSubInvoiceHTLCsRow for rows.Next() { - var i AmpInvoiceHtlc + var i FetchAMPSubInvoiceHTLCsRow if err := rows.Scan( &i.SetID, - &i.HtlcID, - &i.InvoiceID, &i.RootShare, &i.ChildIndex, &i.Hash, &i.Preimage, + &i.ID, + &i.ChanID, + &i.HtlcID, + &i.AmountMsat, + &i.TotalMppMsat, + &i.AcceptHeight, + &i.AcceptTime, + &i.ExpiryHeight, + &i.State, + &i.ResolveTime, + &i.InvoiceID, ); err != nil { return nil, err } @@ -86,29 +88,37 @@ func (q *Queries) GetAMPInvoiceHTLCsByInvoiceID(ctx context.Context, invoiceID i return items, nil } -const getAMPInvoiceHTLCsBySetID = `-- name: GetAMPInvoiceHTLCsBySetID :many -SELECT set_id, htlc_id, invoice_id, root_share, child_index, hash, preimage -FROM amp_invoice_htlcs -WHERE set_id = $1 +const fetchAMPSubInvoices = `-- name: FetchAMPSubInvoices :many +SELECT set_id, state, created_at, settled_at, settle_index, invoice_id +FROM amp_sub_invoices +WHERE invoice_id = $1 +AND ( + set_id = $2 OR + $2 IS NULL +) ` -func (q *Queries) GetAMPInvoiceHTLCsBySetID(ctx context.Context, setID []byte) ([]AmpInvoiceHtlc, error) { - rows, err := q.db.QueryContext(ctx, getAMPInvoiceHTLCsBySetID, setID) +type FetchAMPSubInvoicesParams struct { + InvoiceID int64 + SetID []byte +} + +func (q *Queries) FetchAMPSubInvoices(ctx context.Context, arg FetchAMPSubInvoicesParams) ([]AmpSubInvoice, error) { + rows, err := q.db.QueryContext(ctx, fetchAMPSubInvoices, arg.InvoiceID, arg.SetID) if err != nil { return nil, err } defer rows.Close() - var items []AmpInvoiceHtlc + var items []AmpSubInvoice for rows.Next() { - var i AmpInvoiceHtlc + var i AmpSubInvoice if err := rows.Scan( &i.SetID, - &i.HtlcID, + &i.State, + &i.CreatedAt, + &i.SettledAt, + &i.SettleIndex, &i.InvoiceID, - &i.RootShare, - &i.ChildIndex, - &i.Hash, - &i.Preimage, ); err != nil { return nil, err } @@ -123,28 +133,84 @@ func (q *Queries) GetAMPInvoiceHTLCsBySetID(ctx context.Context, setID []byte) ( return items, nil } -const getSetIDHTLCsCustomRecords = `-- name: GetSetIDHTLCsCustomRecords :many -SELECT ihcr.htlc_id, key, value -FROM amp_invoice_htlcs aih JOIN invoice_htlc_custom_records ihcr ON aih.id=ihcr.htlc_id -WHERE aih.set_id = $1 +const fetchSettledAMPSubInvoices = `-- name: FetchSettledAMPSubInvoices :many +SELECT + a.set_id, + a.settle_index as amp_settle_index, + a.settled_at as amp_settled_at, + i.id, i.hash, i.preimage, i.settle_index, i.settled_at, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.payment_request_hash, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at +FROM amp_sub_invoices a +INNER JOIN invoices i ON a.invoice_id = i.id +WHERE ( + a.settle_index >= $1 OR + $1 IS NULL +) AND ( + a.settle_index <= $2 OR + $2 IS NULL +) ` -type GetSetIDHTLCsCustomRecordsRow struct { - HtlcID int64 - Key int64 - Value []byte +type FetchSettledAMPSubInvoicesParams struct { + SettleIndexGet sql.NullInt64 + SettleIndexLet sql.NullInt64 } -func (q *Queries) GetSetIDHTLCsCustomRecords(ctx context.Context, setID []byte) ([]GetSetIDHTLCsCustomRecordsRow, error) { - rows, err := q.db.QueryContext(ctx, getSetIDHTLCsCustomRecords, setID) +type FetchSettledAMPSubInvoicesRow struct { + SetID []byte + AmpSettleIndex sql.NullInt64 + AmpSettledAt sql.NullTime + ID int64 + Hash []byte + Preimage []byte + SettleIndex sql.NullInt64 + SettledAt sql.NullTime + Memo sql.NullString + AmountMsat int64 + CltvDelta sql.NullInt32 + Expiry int32 + PaymentAddr []byte + PaymentRequest sql.NullString + PaymentRequestHash []byte + State int16 + AmountPaidMsat int64 + IsAmp bool + IsHodl bool + IsKeysend bool + CreatedAt time.Time +} + +func (q *Queries) FetchSettledAMPSubInvoices(ctx context.Context, arg FetchSettledAMPSubInvoicesParams) ([]FetchSettledAMPSubInvoicesRow, error) { + rows, err := q.db.QueryContext(ctx, fetchSettledAMPSubInvoices, arg.SettleIndexGet, arg.SettleIndexLet) if err != nil { return nil, err } defer rows.Close() - var items []GetSetIDHTLCsCustomRecordsRow + var items []FetchSettledAMPSubInvoicesRow for rows.Next() { - var i GetSetIDHTLCsCustomRecordsRow - if err := rows.Scan(&i.HtlcID, &i.Key, &i.Value); err != nil { + var i FetchSettledAMPSubInvoicesRow + if err := rows.Scan( + &i.SetID, + &i.AmpSettleIndex, + &i.AmpSettledAt, + &i.ID, + &i.Hash, + &i.Preimage, + &i.SettleIndex, + &i.SettledAt, + &i.Memo, + &i.AmountMsat, + &i.CltvDelta, + &i.Expiry, + &i.PaymentAddr, + &i.PaymentRequest, + &i.PaymentRequestHash, + &i.State, + &i.AmountPaidMsat, + &i.IsAmp, + &i.IsHodl, + &i.IsKeysend, + &i.CreatedAt, + ); err != nil { return nil, err } items = append(items, i) @@ -158,15 +224,27 @@ func (q *Queries) GetSetIDHTLCsCustomRecords(ctx context.Context, setID []byte) return items, nil } -const insertAMPInvoiceHTLC = `-- name: InsertAMPInvoiceHTLC :exec -INSERT INTO amp_invoice_htlcs ( - set_id, htlc_id, root_share, child_index, hash, preimage +const getAMPInvoiceID = `-- name: GetAMPInvoiceID :one +SELECT invoice_id FROM amp_sub_invoices WHERE set_id = $1 +` + +func (q *Queries) GetAMPInvoiceID(ctx context.Context, setID []byte) (int64, error) { + row := q.db.QueryRowContext(ctx, getAMPInvoiceID, setID) + var invoice_id int64 + err := row.Scan(&invoice_id) + return invoice_id, err +} + +const insertAMPSubInvoiceHTLC = `-- name: InsertAMPSubInvoiceHTLC :exec +INSERT INTO amp_sub_invoice_htlcs ( + invoice_id, set_id, htlc_id, root_share, child_index, hash, preimage ) VALUES ( - $1, $2, $3, $4, $5, $6 + $1, $2, $3, $4, $5, $6, $7 ) ` -type InsertAMPInvoiceHTLCParams struct { +type InsertAMPSubInvoiceHTLCParams struct { + InvoiceID int64 SetID []byte HtlcID int64 RootShare []byte @@ -175,8 +253,9 @@ type InsertAMPInvoiceHTLCParams struct { Preimage []byte } -func (q *Queries) InsertAMPInvoiceHTLC(ctx context.Context, arg InsertAMPInvoiceHTLCParams) error { - _, err := q.db.ExecContext(ctx, insertAMPInvoiceHTLC, +func (q *Queries) InsertAMPSubInvoiceHTLC(ctx context.Context, arg InsertAMPSubInvoiceHTLCParams) error { + _, err := q.db.ExecContext(ctx, insertAMPSubInvoiceHTLC, + arg.InvoiceID, arg.SetID, arg.HtlcID, arg.RootShare, @@ -187,140 +266,75 @@ func (q *Queries) InsertAMPInvoiceHTLC(ctx context.Context, arg InsertAMPInvoice return err } -const insertAMPInvoicePayment = `-- name: InsertAMPInvoicePayment :exec -INSERT INTO amp_invoice_payments ( - set_id, state, created_at, settled_index, invoice_id -) VALUES ( - $1, $2, $3, $4, $5 +const updateAMPSubInvoiceHTLCPreimage = `-- name: UpdateAMPSubInvoiceHTLCPreimage :execresult +UPDATE amp_sub_invoice_htlcs AS a +SET preimage = $4 +WHERE a.invoice_id = $1 AND a.set_id = $2 AND a.htlc_id = ( + SELECT id FROM invoice_htlcs AS i WHERE i.htlc_id = $3 ) ` -type InsertAMPInvoicePaymentParams struct { - SetID []byte - State int16 - CreatedAt time.Time - SettledIndex sql.NullInt32 - InvoiceID int32 +type UpdateAMPSubInvoiceHTLCPreimageParams struct { + InvoiceID int64 + SetID []byte + HtlcID int64 + Preimage []byte } -func (q *Queries) InsertAMPInvoicePayment(ctx context.Context, arg InsertAMPInvoicePaymentParams) error { - _, err := q.db.ExecContext(ctx, insertAMPInvoicePayment, +func (q *Queries) UpdateAMPSubInvoiceHTLCPreimage(ctx context.Context, arg UpdateAMPSubInvoiceHTLCPreimageParams) (sql.Result, error) { + return q.db.ExecContext(ctx, updateAMPSubInvoiceHTLCPreimage, + arg.InvoiceID, + arg.SetID, + arg.HtlcID, + arg.Preimage, + ) +} + +const updateAMPSubInvoiceState = `-- name: UpdateAMPSubInvoiceState :exec +UPDATE amp_sub_invoices +SET state = $2, + settle_index = COALESCE(settle_index, $3), + settled_at = COALESCE(settled_at, $4) +WHERE set_id = $1 +` + +type UpdateAMPSubInvoiceStateParams struct { + SetID []byte + State int16 + SettleIndex sql.NullInt64 + SettledAt sql.NullTime +} + +func (q *Queries) UpdateAMPSubInvoiceState(ctx context.Context, arg UpdateAMPSubInvoiceStateParams) error { + _, err := q.db.ExecContext(ctx, updateAMPSubInvoiceState, + arg.SetID, + arg.State, + arg.SettleIndex, + arg.SettledAt, + ) + return err +} + +const upsertAMPSubInvoice = `-- name: UpsertAMPSubInvoice :execresult +INSERT INTO amp_sub_invoices ( + set_id, state, created_at, invoice_id +) VALUES ( + $1, $2, $3, $4 +) ON CONFLICT (set_id, invoice_id) DO NOTHING +` + +type UpsertAMPSubInvoiceParams struct { + SetID []byte + State int16 + CreatedAt time.Time + InvoiceID int64 +} + +func (q *Queries) UpsertAMPSubInvoice(ctx context.Context, arg UpsertAMPSubInvoiceParams) (sql.Result, error) { + return q.db.ExecContext(ctx, upsertAMPSubInvoice, arg.SetID, arg.State, arg.CreatedAt, - arg.SettledIndex, arg.InvoiceID, ) - return err -} - -const selectAMPInvoicePayments = `-- name: SelectAMPInvoicePayments :many -SELECT aip.set_id, aip.state, aip.created_at, aip.settled_index, aip.invoice_id, ip.id, ip.settled_at, ip.amount_paid_msat, ip.invoice_id -FROM amp_invoice_payments aip LEFT JOIN invoice_payments ip ON aip.settled_index = ip.id -WHERE ( - set_id = $1 OR - $1 IS NULL -) AND ( - aip.settled_index = $2 OR - $2 IS NULL -) AND ( - aip.invoice_id = $3 OR - $3 IS NULL -) -` - -type SelectAMPInvoicePaymentsParams struct { - SetID []byte - SettledIndex sql.NullInt32 - InvoiceID sql.NullInt32 -} - -type SelectAMPInvoicePaymentsRow struct { - SetID []byte - State int16 - CreatedAt time.Time - SettledIndex sql.NullInt32 - InvoiceID int32 - ID sql.NullInt32 - SettledAt sql.NullTime - AmountPaidMsat sql.NullInt64 - InvoiceID_2 sql.NullInt32 -} - -func (q *Queries) SelectAMPInvoicePayments(ctx context.Context, arg SelectAMPInvoicePaymentsParams) ([]SelectAMPInvoicePaymentsRow, error) { - rows, err := q.db.QueryContext(ctx, selectAMPInvoicePayments, arg.SetID, arg.SettledIndex, arg.InvoiceID) - if err != nil { - return nil, err - } - defer rows.Close() - var items []SelectAMPInvoicePaymentsRow - for rows.Next() { - var i SelectAMPInvoicePaymentsRow - if err := rows.Scan( - &i.SetID, - &i.State, - &i.CreatedAt, - &i.SettledIndex, - &i.InvoiceID, - &i.ID, - &i.SettledAt, - &i.AmountPaidMsat, - &i.InvoiceID_2, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - -const updateAMPInvoiceHTLC = `-- name: UpdateAMPInvoiceHTLC :exec -UPDATE amp_invoice_htlcs -SET preimage = $1 -WHERE htlc_id = $2 -` - -type UpdateAMPInvoiceHTLCParams struct { - Preimage []byte - HtlcID int64 -} - -func (q *Queries) UpdateAMPInvoiceHTLC(ctx context.Context, arg UpdateAMPInvoiceHTLCParams) error { - _, err := q.db.ExecContext(ctx, updateAMPInvoiceHTLC, arg.Preimage, arg.HtlcID) - return err -} - -const updateAMPPayment = `-- name: UpdateAMPPayment :exec -UPDATE amp_invoice_payments -SET state = $1, settled_index = $2 -WHERE state = 0 AND ( - set_id = $3 OR - $3 IS NULL -) AND ( - invoice_id = $4 OR - $4 IS NULL -) -` - -type UpdateAMPPaymentParams struct { - State int16 - SettledIndex sql.NullInt32 - SetID []byte - InvoiceID sql.NullInt32 -} - -func (q *Queries) UpdateAMPPayment(ctx context.Context, arg UpdateAMPPaymentParams) error { - _, err := q.db.ExecContext(ctx, updateAMPPayment, - arg.State, - arg.SettledIndex, - arg.SetID, - arg.InvoiceID, - ) - return err } diff --git a/sqldb/sqlc/invoice_events.sql.go b/sqldb/sqlc/invoice_events.sql.go index b0623fc1e..983dfbb15 100644 --- a/sqldb/sqlc/invoice_events.sql.go +++ b/sqldb/sqlc/invoice_events.sql.go @@ -7,122 +7,116 @@ package sqlc import ( "context" - "database/sql" "time" ) -const deleteInvoiceEvents = `-- name: DeleteInvoiceEvents :exec -DELETE -FROM invoice_events -WHERE invoice_id = $1 -` - -func (q *Queries) DeleteInvoiceEvents(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteInvoiceEvents, invoiceID) - return err -} - -const insertInvoiceEvent = `-- name: InsertInvoiceEvent :exec +const onAMPSubInvoiceCanceled = `-- name: OnAMPSubInvoiceCanceled :exec INSERT INTO invoice_events ( - created_at, invoice_id, htlc_id, set_id, event_type, event_metadata + added_at, event_type, invoice_id, set_id ) VALUES ( - $1, $2, $3, $4, $5, $6 + $1, 4, $2, $3 ) ` -type InsertInvoiceEventParams struct { - CreatedAt time.Time - InvoiceID int32 - HtlcID sql.NullInt64 - SetID []byte - EventType int32 - EventMetadata []byte +type OnAMPSubInvoiceCanceledParams struct { + AddedAt time.Time + InvoiceID int64 + SetID []byte } -func (q *Queries) InsertInvoiceEvent(ctx context.Context, arg InsertInvoiceEventParams) error { - _, err := q.db.ExecContext(ctx, insertInvoiceEvent, - arg.CreatedAt, - arg.InvoiceID, - arg.HtlcID, - arg.SetID, - arg.EventType, - arg.EventMetadata, - ) +func (q *Queries) OnAMPSubInvoiceCanceled(ctx context.Context, arg OnAMPSubInvoiceCanceledParams) error { + _, err := q.db.ExecContext(ctx, onAMPSubInvoiceCanceled, arg.AddedAt, arg.InvoiceID, arg.SetID) return err } -const selectInvoiceEvents = `-- name: SelectInvoiceEvents :many -SELECT id, created_at, invoice_id, htlc_id, set_id, event_type, event_metadata -FROM invoice_events -WHERE ( - invoice_id = $1 OR - $1 IS NULL -) AND ( - htlc_id = $2 OR - $2 IS NULL -) AND ( - set_id = $3 OR - $3 IS NULL -) AND ( - event_type = $4 OR - $4 IS NULL -) AND ( - created_at >= $5 OR - $5 IS NULL -) AND ( - created_at <= $6 OR - $6 IS NULL -) -LIMIT $8 OFFSET $7 +const onAMPSubInvoiceCreated = `-- name: OnAMPSubInvoiceCreated :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id, set_id +) VALUES ( + $1, 3, $2, $3 +) ` -type SelectInvoiceEventsParams struct { - InvoiceID sql.NullInt32 - HtlcID sql.NullInt64 - SetID []byte - EventType sql.NullInt32 - CreatedAfter sql.NullTime - CreatedBefore sql.NullTime - NumOffset int32 - NumLimit int32 +type OnAMPSubInvoiceCreatedParams struct { + AddedAt time.Time + InvoiceID int64 + SetID []byte } -func (q *Queries) SelectInvoiceEvents(ctx context.Context, arg SelectInvoiceEventsParams) ([]InvoiceEvent, error) { - rows, err := q.db.QueryContext(ctx, selectInvoiceEvents, - arg.InvoiceID, - arg.HtlcID, - arg.SetID, - arg.EventType, - arg.CreatedAfter, - arg.CreatedBefore, - arg.NumOffset, - arg.NumLimit, - ) - if err != nil { - return nil, err - } - defer rows.Close() - var items []InvoiceEvent - for rows.Next() { - var i InvoiceEvent - if err := rows.Scan( - &i.ID, - &i.CreatedAt, - &i.InvoiceID, - &i.HtlcID, - &i.SetID, - &i.EventType, - &i.EventMetadata, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil +func (q *Queries) OnAMPSubInvoiceCreated(ctx context.Context, arg OnAMPSubInvoiceCreatedParams) error { + _, err := q.db.ExecContext(ctx, onAMPSubInvoiceCreated, arg.AddedAt, arg.InvoiceID, arg.SetID) + return err +} + +const onAMPSubInvoiceSettled = `-- name: OnAMPSubInvoiceSettled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id, set_id +) VALUES ( + $1, 5, $2, $3 +) +` + +type OnAMPSubInvoiceSettledParams struct { + AddedAt time.Time + InvoiceID int64 + SetID []byte +} + +func (q *Queries) OnAMPSubInvoiceSettled(ctx context.Context, arg OnAMPSubInvoiceSettledParams) error { + _, err := q.db.ExecContext(ctx, onAMPSubInvoiceSettled, arg.AddedAt, arg.InvoiceID, arg.SetID) + return err +} + +const onInvoiceCanceled = `-- name: OnInvoiceCanceled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id +) VALUES ( + $1, 1, $2 +) +` + +type OnInvoiceCanceledParams struct { + AddedAt time.Time + InvoiceID int64 +} + +func (q *Queries) OnInvoiceCanceled(ctx context.Context, arg OnInvoiceCanceledParams) error { + _, err := q.db.ExecContext(ctx, onInvoiceCanceled, arg.AddedAt, arg.InvoiceID) + return err +} + +const onInvoiceCreated = `-- name: OnInvoiceCreated :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id +) VALUES ( + $1, 0, $2 +) +` + +type OnInvoiceCreatedParams struct { + AddedAt time.Time + InvoiceID int64 +} + +func (q *Queries) OnInvoiceCreated(ctx context.Context, arg OnInvoiceCreatedParams) error { + _, err := q.db.ExecContext(ctx, onInvoiceCreated, arg.AddedAt, arg.InvoiceID) + return err +} + +const onInvoiceSettled = `-- name: OnInvoiceSettled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id +) VALUES ( + $1, 2, $2 +) +` + +type OnInvoiceSettledParams struct { + AddedAt time.Time + InvoiceID int64 +} + +func (q *Queries) OnInvoiceSettled(ctx context.Context, arg OnInvoiceSettledParams) error { + _, err := q.db.ExecContext(ctx, onInvoiceSettled, arg.AddedAt, arg.InvoiceID) + return err } diff --git a/sqldb/sqlc/invoices.sql.go b/sqldb/sqlc/invoices.sql.go index 927905c96..09ca1a1e2 100644 --- a/sqldb/sqlc/invoices.sql.go +++ b/sqldb/sqlc/invoices.sql.go @@ -11,7 +11,17 @@ import ( "time" ) -const deleteInvoice = `-- name: DeleteInvoice :exec +const deleteCanceledInvoices = `-- name: DeleteCanceledInvoices :execresult +DELETE +FROM invoices +WHERE state = 2 +` + +func (q *Queries) DeleteCanceledInvoices(ctx context.Context) (sql.Result, error) { + return q.db.ExecContext(ctx, deleteCanceledInvoices) +} + +const deleteInvoice = `-- name: DeleteInvoice :execresult DELETE FROM invoices WHERE ( @@ -21,7 +31,7 @@ WHERE ( hash = $2 OR $2 IS NULL ) AND ( - preimage = $3 OR + settle_index = $3 OR $3 IS NULL ) AND ( payment_addr = $4 OR @@ -30,174 +40,24 @@ WHERE ( ` type DeleteInvoiceParams struct { - AddIndex sql.NullInt32 + AddIndex sql.NullInt64 Hash []byte - Preimage []byte + SettleIndex sql.NullInt64 PaymentAddr []byte } -func (q *Queries) DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) error { - _, err := q.db.ExecContext(ctx, deleteInvoice, +func (q *Queries) DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) (sql.Result, error) { + return q.db.ExecContext(ctx, deleteInvoice, arg.AddIndex, arg.Hash, - arg.Preimage, + arg.SettleIndex, arg.PaymentAddr, ) - return err -} - -const deleteInvoiceFeatures = `-- name: DeleteInvoiceFeatures :exec -DELETE -FROM invoice_features -WHERE invoice_id = $1 -` - -func (q *Queries) DeleteInvoiceFeatures(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteInvoiceFeatures, invoiceID) - return err -} - -const deleteInvoiceHTLC = `-- name: DeleteInvoiceHTLC :exec -DELETE -FROM invoice_htlcs -WHERE htlc_id = $1 -` - -func (q *Queries) DeleteInvoiceHTLC(ctx context.Context, htlcID int64) error { - _, err := q.db.ExecContext(ctx, deleteInvoiceHTLC, htlcID) - return err -} - -const deleteInvoiceHTLCCustomRecords = `-- name: DeleteInvoiceHTLCCustomRecords :exec -WITH htlc_ids AS ( - SELECT ih.id - FROM invoice_htlcs ih JOIN invoice_htlc_custom_records ihcr ON ih.id=ihcr.htlc_id - WHERE ih.invoice_id = $1 -) -DELETE -FROM invoice_htlc_custom_records -WHERE htlc_id IN (SELECT id FROM htlc_ids) -` - -func (q *Queries) DeleteInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteInvoiceHTLCCustomRecords, invoiceID) - return err -} - -const deleteInvoiceHTLCs = `-- name: DeleteInvoiceHTLCs :exec -DELETE -FROM invoice_htlcs -WHERE invoice_id = $1 -` - -func (q *Queries) DeleteInvoiceHTLCs(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteInvoiceHTLCs, invoiceID) - return err -} - -const filterInvoicePayments = `-- name: FilterInvoicePayments :many -SELECT - ip.id AS settle_index, ip.amount_paid_msat, ip.settled_at AS settle_date, - i.id, i.hash, i.preimage, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at -FROM invoice_payments ip JOIN invoices i ON ip.invoice_id = i.id -WHERE ( - ip.id >= $1 OR - $1 IS NULL -) AND ( - ip.settled_at >= $2 OR - $2 IS NULL -) -ORDER BY - CASE - WHEN $3 = FALSE THEN ip.id - ELSE NULL - END ASC, - CASE - WHEN $3 = TRUE THEN ip.id - ELSE NULL - END DESC -LIMIT $5 OFFSET $4 -` - -type FilterInvoicePaymentsParams struct { - SettleIndexGet sql.NullInt32 - SettledAfter sql.NullTime - Reverse interface{} - NumOffset int32 - NumLimit int32 -} - -type FilterInvoicePaymentsRow struct { - SettleIndex int32 - AmountPaidMsat int64 - SettleDate time.Time - ID int32 - Hash []byte - Preimage []byte - Memo sql.NullString - AmountMsat int64 - CltvDelta sql.NullInt32 - Expiry int32 - PaymentAddr []byte - PaymentRequest sql.NullString - State int16 - AmountPaidMsat_2 int64 - IsAmp bool - IsHodl bool - IsKeysend bool - CreatedAt time.Time -} - -func (q *Queries) FilterInvoicePayments(ctx context.Context, arg FilterInvoicePaymentsParams) ([]FilterInvoicePaymentsRow, error) { - rows, err := q.db.QueryContext(ctx, filterInvoicePayments, - arg.SettleIndexGet, - arg.SettledAfter, - arg.Reverse, - arg.NumOffset, - arg.NumLimit, - ) - if err != nil { - return nil, err - } - defer rows.Close() - var items []FilterInvoicePaymentsRow - for rows.Next() { - var i FilterInvoicePaymentsRow - if err := rows.Scan( - &i.SettleIndex, - &i.AmountPaidMsat, - &i.SettleDate, - &i.ID, - &i.Hash, - &i.Preimage, - &i.Memo, - &i.AmountMsat, - &i.CltvDelta, - &i.Expiry, - &i.PaymentAddr, - &i.PaymentRequest, - &i.State, - &i.AmountPaidMsat_2, - &i.IsAmp, - &i.IsHodl, - &i.IsKeysend, - &i.CreatedAt, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil } const filterInvoices = `-- name: FilterInvoices :many -SELECT id, hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, created_at +SELECT + invoices.id, invoices.hash, invoices.preimage, invoices.settle_index, invoices.settled_at, invoices.memo, invoices.amount_msat, invoices.cltv_delta, invoices.expiry, invoices.payment_addr, invoices.payment_request, invoices.payment_request_hash, invoices.state, invoices.amount_paid_msat, invoices.is_amp, invoices.is_hodl, invoices.is_keysend, invoices.created_at FROM invoices WHERE ( id >= $1 OR @@ -206,48 +66,58 @@ WHERE ( id <= $2 OR $2 IS NULL ) AND ( - state = $3 OR + settle_index >= $3 OR $3 IS NULL ) AND ( - created_at >= $4 OR + settle_index <= $4 OR $4 IS NULL ) AND ( - created_at <= $5 OR + state = $5 OR $5 IS NULL +) AND ( + created_at >= $6 OR + $6 IS NULL +) AND ( + created_at <= $7 OR + $7 IS NULL ) AND ( CASE - WHEN $6=TRUE THEN (state = 0 OR state = 3) + WHEN $8=TRUE THEN (state = 0 OR state = 3) ELSE TRUE END ) ORDER BY CASE - WHEN $7 = FALSE THEN id + WHEN $9 = FALSE THEN id ELSE NULL END ASC, CASE - WHEN $7 = TRUE THEN id + WHEN $9 = TRUE THEN id ELSE NULL END DESC -LIMIT $9 OFFSET $8 +LIMIT $11 OFFSET $10 ` type FilterInvoicesParams struct { - AddIndexGet sql.NullInt32 - AddIndexLet sql.NullInt32 - State sql.NullInt16 - CreatedAfter sql.NullTime - CreatedBefore sql.NullTime - PendingOnly interface{} - Reverse interface{} - NumOffset int32 - NumLimit int32 + AddIndexGet sql.NullInt64 + AddIndexLet sql.NullInt64 + SettleIndexGet sql.NullInt64 + SettleIndexLet sql.NullInt64 + State sql.NullInt16 + CreatedAfter sql.NullTime + CreatedBefore sql.NullTime + PendingOnly interface{} + Reverse interface{} + NumOffset int32 + NumLimit int32 } func (q *Queries) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) ([]Invoice, error) { rows, err := q.db.QueryContext(ctx, filterInvoices, arg.AddIndexGet, arg.AddIndexLet, + arg.SettleIndexGet, + arg.SettleIndexLet, arg.State, arg.CreatedAfter, arg.CreatedBefore, @@ -267,12 +137,15 @@ func (q *Queries) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) &i.ID, &i.Hash, &i.Preimage, + &i.SettleIndex, + &i.SettledAt, &i.Memo, &i.AmountMsat, &i.CltvDelta, &i.Expiry, &i.PaymentAddr, &i.PaymentRequest, + &i.PaymentRequestHash, &i.State, &i.AmountPaidMsat, &i.IsAmp, @@ -295,29 +168,35 @@ func (q *Queries) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) const getInvoice = `-- name: GetInvoice :many -SELECT id, hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, created_at -FROM invoices +SELECT i.id, i.hash, i.preimage, i.settle_index, i.settled_at, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.payment_request_hash, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at +FROM invoices i +LEFT JOIN amp_sub_invoices a on i.id = a.invoice_id WHERE ( - id = $1 OR + i.id = $1 OR $1 IS NULL ) AND ( - hash = $2 OR + i.hash = $2 OR $2 IS NULL ) AND ( - preimage = $3 OR + i.preimage = $3 OR $3 IS NULL ) AND ( - payment_addr = $4 OR + i.payment_addr = $4 OR $4 IS NULL +) AND ( + a.set_id = $5 OR + $5 IS NULL ) +GROUP BY i.id LIMIT 2 ` type GetInvoiceParams struct { - AddIndex sql.NullInt32 + AddIndex sql.NullInt64 Hash []byte Preimage []byte PaymentAddr []byte + SetID []byte } // This method may return more than one invoice if filter using multiple fields @@ -329,6 +208,7 @@ func (q *Queries) GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoi arg.Hash, arg.Preimage, arg.PaymentAddr, + arg.SetID, ) if err != nil { return nil, err @@ -341,12 +221,15 @@ func (q *Queries) GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoi &i.ID, &i.Hash, &i.Preimage, + &i.SettleIndex, + &i.SettledAt, &i.Memo, &i.AmountMsat, &i.CltvDelta, &i.Expiry, &i.PaymentAddr, &i.PaymentRequest, + &i.PaymentRequestHash, &i.State, &i.AmountPaidMsat, &i.IsAmp, @@ -373,7 +256,7 @@ FROM invoice_features WHERE invoice_id = $1 ` -func (q *Queries) GetInvoiceFeatures(ctx context.Context, invoiceID int32) ([]InvoiceFeature, error) { +func (q *Queries) GetInvoiceFeatures(ctx context.Context, invoiceID int64) ([]InvoiceFeature, error) { rows, err := q.db.QueryContext(ctx, getInvoiceFeatures, invoiceID) if err != nil { return nil, err @@ -408,7 +291,7 @@ type GetInvoiceHTLCCustomRecordsRow struct { Value []byte } -func (q *Queries) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) ([]GetInvoiceHTLCCustomRecordsRow, error) { +func (q *Queries) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error) { rows, err := q.db.QueryContext(ctx, getInvoiceHTLCCustomRecords, invoiceID) if err != nil { return nil, err @@ -432,12 +315,12 @@ func (q *Queries) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int } const getInvoiceHTLCs = `-- name: GetInvoiceHTLCs :many -SELECT id, htlc_id, chan_id, amount_msat, total_mpp_msat, accept_height, accept_time, expiry_height, state, resolve_time, invoice_id +SELECT id, chan_id, htlc_id, amount_msat, total_mpp_msat, accept_height, accept_time, expiry_height, state, resolve_time, invoice_id FROM invoice_htlcs WHERE invoice_id = $1 ` -func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]InvoiceHtlc, error) { +func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int64) ([]InvoiceHtlc, error) { rows, err := q.db.QueryContext(ctx, getInvoiceHTLCs, invoiceID) if err != nil { return nil, err @@ -448,8 +331,8 @@ func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]Invoi var i InvoiceHtlc if err := rows.Scan( &i.ID, - &i.HtlcID, &i.ChanID, + &i.HtlcID, &i.AmountMsat, &i.TotalMppMsat, &i.AcceptHeight, @@ -472,68 +355,35 @@ func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]Invoi return items, nil } -const getInvoicePayments = `-- name: GetInvoicePayments :many -SELECT id, settled_at, amount_paid_msat, invoice_id -FROM invoice_payments -WHERE invoice_id = $1 -` - -func (q *Queries) GetInvoicePayments(ctx context.Context, invoiceID int32) ([]InvoicePayment, error) { - rows, err := q.db.QueryContext(ctx, getInvoicePayments, invoiceID) - if err != nil { - return nil, err - } - defer rows.Close() - var items []InvoicePayment - for rows.Next() { - var i InvoicePayment - if err := rows.Scan( - &i.ID, - &i.SettledAt, - &i.AmountPaidMsat, - &i.InvoiceID, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const insertInvoice = `-- name: InsertInvoice :one INSERT INTO invoices ( hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, - payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, - created_at + payment_request, payment_request_hash, state, amount_paid_msat, is_amp, + is_hodl, is_keysend, created_at ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15 ) RETURNING id ` type InsertInvoiceParams struct { - Hash []byte - Preimage []byte - Memo sql.NullString - AmountMsat int64 - CltvDelta sql.NullInt32 - Expiry int32 - PaymentAddr []byte - PaymentRequest sql.NullString - State int16 - AmountPaidMsat int64 - IsAmp bool - IsHodl bool - IsKeysend bool - CreatedAt time.Time + Hash []byte + Preimage []byte + Memo sql.NullString + AmountMsat int64 + CltvDelta sql.NullInt32 + Expiry int32 + PaymentAddr []byte + PaymentRequest sql.NullString + PaymentRequestHash []byte + State int16 + AmountPaidMsat int64 + IsAmp bool + IsHodl bool + IsKeysend bool + CreatedAt time.Time } -func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int32, error) { +func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int64, error) { row := q.db.QueryRowContext(ctx, insertInvoice, arg.Hash, arg.Preimage, @@ -543,6 +393,7 @@ func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (i arg.Expiry, arg.PaymentAddr, arg.PaymentRequest, + arg.PaymentRequestHash, arg.State, arg.AmountPaidMsat, arg.IsAmp, @@ -550,7 +401,7 @@ func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (i arg.IsKeysend, arg.CreatedAt, ) - var id int32 + var id int64 err := row.Scan(&id) return id, err } @@ -564,7 +415,7 @@ INSERT INTO invoice_features ( ` type InsertInvoiceFeatureParams struct { - InvoiceID int32 + InvoiceID int64 Feature int32 } @@ -573,13 +424,13 @@ func (q *Queries) InsertInvoiceFeature(ctx context.Context, arg InsertInvoiceFea return err } -const insertInvoiceHTLC = `-- name: InsertInvoiceHTLC :exec +const insertInvoiceHTLC = `-- name: InsertInvoiceHTLC :one INSERT INTO invoice_htlcs ( htlc_id, chan_id, amount_msat, total_mpp_msat, accept_height, accept_time, expiry_height, state, resolve_time, invoice_id ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 -) +) RETURNING id ` type InsertInvoiceHTLCParams struct { @@ -592,11 +443,11 @@ type InsertInvoiceHTLCParams struct { ExpiryHeight int32 State int16 ResolveTime sql.NullTime - InvoiceID int32 + InvoiceID int64 } -func (q *Queries) InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) error { - _, err := q.db.ExecContext(ctx, insertInvoiceHTLC, +func (q *Queries) InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertInvoiceHTLC, arg.HtlcID, arg.ChanID, arg.AmountMsat, @@ -608,7 +459,9 @@ func (q *Queries) InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCPa arg.ResolveTime, arg.InvoiceID, ) - return err + var id int64 + err := row.Scan(&id) + return id, err } const insertInvoiceHTLCCustomRecord = `-- name: InsertInvoiceHTLCCustomRecord :exec @@ -630,64 +483,56 @@ func (q *Queries) InsertInvoiceHTLCCustomRecord(ctx context.Context, arg InsertI return err } -const insertInvoicePayment = `-- name: InsertInvoicePayment :one -INSERT INTO invoice_payments ( - invoice_id, amount_paid_msat, settled_at -) VALUES ( - $1, $2, $3 -) RETURNING id +const nextInvoiceSettleIndex = `-- name: NextInvoiceSettleIndex :one +UPDATE invoice_sequences SET current_value = current_value + 1 +WHERE name = 'settle_index' +RETURNING current_value ` -type InsertInvoicePaymentParams struct { - InvoiceID int32 - AmountPaidMsat int64 - SettledAt time.Time +func (q *Queries) NextInvoiceSettleIndex(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, nextInvoiceSettleIndex) + var current_value int64 + err := row.Scan(¤t_value) + return current_value, err } -func (q *Queries) InsertInvoicePayment(ctx context.Context, arg InsertInvoicePaymentParams) (int32, error) { - row := q.db.QueryRowContext(ctx, insertInvoicePayment, arg.InvoiceID, arg.AmountPaidMsat, arg.SettledAt) - var id int32 - err := row.Scan(&id) - return id, err -} - -const updateInvoice = `-- name: UpdateInvoice :exec -UPDATE invoices -SET preimage=$2, state=$3, amount_paid_msat=$4 -WHERE id=$1 +const updateInvoiceAmountPaid = `-- name: UpdateInvoiceAmountPaid :execresult +UPDATE invoices +SET amount_paid_msat = $2 +WHERE id = $1 ` -type UpdateInvoiceParams struct { - ID int32 - Preimage []byte - State int16 +type UpdateInvoiceAmountPaidParams struct { + ID int64 AmountPaidMsat int64 } -func (q *Queries) UpdateInvoice(ctx context.Context, arg UpdateInvoiceParams) error { - _, err := q.db.ExecContext(ctx, updateInvoice, - arg.ID, - arg.Preimage, - arg.State, - arg.AmountPaidMsat, - ) - return err +func (q *Queries) UpdateInvoiceAmountPaid(ctx context.Context, arg UpdateInvoiceAmountPaidParams) (sql.Result, error) { + return q.db.ExecContext(ctx, updateInvoiceAmountPaid, arg.ID, arg.AmountPaidMsat) } const updateInvoiceHTLC = `-- name: UpdateInvoiceHTLC :exec UPDATE invoice_htlcs -SET state=$2, resolve_time=$3 -WHERE id = $1 +SET state=$4, resolve_time=$5 +WHERE htlc_id = $1 AND chan_id = $2 AND invoice_id = $3 ` type UpdateInvoiceHTLCParams struct { - ID int32 + HtlcID int64 + ChanID string + InvoiceID int64 State int16 ResolveTime sql.NullTime } func (q *Queries) UpdateInvoiceHTLC(ctx context.Context, arg UpdateInvoiceHTLCParams) error { - _, err := q.db.ExecContext(ctx, updateInvoiceHTLC, arg.ID, arg.State, arg.ResolveTime) + _, err := q.db.ExecContext(ctx, updateInvoiceHTLC, + arg.HtlcID, + arg.ChanID, + arg.InvoiceID, + arg.State, + arg.ResolveTime, + ) return err } @@ -698,7 +543,7 @@ WHERE invoice_id = $1 AND resolve_time IS NULL ` type UpdateInvoiceHTLCsParams struct { - InvoiceID int32 + InvoiceID int64 State int16 ResolveTime sql.NullTime } @@ -707,3 +552,30 @@ func (q *Queries) UpdateInvoiceHTLCs(ctx context.Context, arg UpdateInvoiceHTLCs _, err := q.db.ExecContext(ctx, updateInvoiceHTLCs, arg.InvoiceID, arg.State, arg.ResolveTime) return err } + +const updateInvoiceState = `-- name: UpdateInvoiceState :execresult +UPDATE invoices +SET state = $2, + preimage = COALESCE(preimage, $3), + settle_index = COALESCE(settle_index, $4), + settled_at = COALESCE(settled_at, $5) +WHERE id = $1 +` + +type UpdateInvoiceStateParams struct { + ID int64 + State int16 + Preimage []byte + SettleIndex sql.NullInt64 + SettledAt sql.NullTime +} + +func (q *Queries) UpdateInvoiceState(ctx context.Context, arg UpdateInvoiceStateParams) (sql.Result, error) { + return q.db.ExecContext(ctx, updateInvoiceState, + arg.ID, + arg.State, + arg.Preimage, + arg.SettleIndex, + arg.SettledAt, + ) +} diff --git a/sqldb/sqlc/migrations/000001_invoices.up.sql b/sqldb/sqlc/migrations/000001_invoices.up.sql index 32b70b9fa..e4c80a26f 100644 --- a/sqldb/sqlc/migrations/000001_invoices.up.sql +++ b/sqldb/sqlc/migrations/000001_invoices.up.sql @@ -1,7 +1,17 @@ +-- sequences contains all sequences used for invoices. +CREATE TABLE invoice_sequences ( + name TEXT PRIMARY KEY, + current_value BIGINT NOT NULL +); + +-- Initialize a sequence for the settled index used to track invoice settlement +-- to remain compatible with the legacy channeldb implementation. +INSERT INTO invoice_sequences(name, current_value) VALUES ('settle_index', 0); + -- invoices table contains all the information shared by all the invoice types. CREATE TABLE IF NOT EXISTS invoices ( -- The id of the invoice. Translates to the AddIndex. - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, -- The hash for this invoice. The invoice hash will always identify that -- invoice. @@ -10,6 +20,14 @@ CREATE TABLE IF NOT EXISTS invoices ( -- The preimage for the hash in this invoice. Some invoices may have this -- field empty, like unsettled hodl invoices or AMP invoices. preimage BLOB, + + -- If settled, the index is set to the current_value+1 of the settle_index + -- seuqence in the invoice_sequences table. If not settled, then it is set + -- to NULL. + settle_index BIGINT, + + -- When the invoice was settled. + settled_at TIMESTAMP, -- An optional memo to attach along with the invoice. memo TEXT, @@ -31,7 +49,12 @@ CREATE TABLE IF NOT EXISTS invoices ( -- The encoded payment request for this invoice. Some invoice types may -- not have leave this empty, like Keysends. - payment_request TEXT UNIQUE, + payment_request TEXT, + + -- Holds the hash of the payment request. This field is used to ensure that + -- there are no duplicates in the database. This trick is needed because + -- PostgreSQL has a limitiation of 2712 bytes for unique index rows. + payment_request_hash BLOB UNIQUE, -- The invoice state. state SMALLINT NOT NULL, @@ -58,6 +81,7 @@ CREATE INDEX IF NOT EXISTS invoices_preimage_idx ON invoices(preimage); CREATE INDEX IF NOT EXISTS invoices_payment_addr_idx ON invoices(payment_addr); CREATE INDEX IF NOT EXISTS invoices_state_idx ON invoices(state); CREATE INDEX IF NOT EXISTS invoices_created_at_idx ON invoices(created_at); +CREATE INDEX IF NOT EXISTS invoices_settled_at_idx ON invoices(settled_at); -- invoice_features contains the feature bits of an invoice. CREATE TABLE IF NOT EXISTS invoice_features ( @@ -65,7 +89,7 @@ CREATE TABLE IF NOT EXISTS invoice_features ( feature INTEGER NOT NULL, -- The invoice id this feature belongs to. - invoice_id INTEGER NOT NULL REFERENCES invoices(id), + invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, -- The feature bit is unique per invoice. UNIQUE (feature, invoice_id) @@ -78,15 +102,15 @@ CREATE INDEX IF NOT EXISTS invoice_feature_invoice_id_idx ON invoice_features(in CREATE TABLE IF NOT EXISTS invoice_htlcs ( -- The id for this htlc. Used in foreign keys instead of the -- htlc_id/chan_id combination. - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, + + -- Short chan id indicating the htlc's origin. uint64 stored as text. + chan_id TEXT NOT NULL, -- The uint64 htlc id. This field is a counter so it is safe to store it as -- int64 in the database. The application layer must check that there is no -- overflow when storing/loading this column. htlc_id BIGINT NOT NULL, - - -- Short chan id indicating the htlc's origin. uint64 stored as text. - chan_id TEXT NOT NULL, -- The htlc's amount in millisatoshis. amount_msat BIGINT NOT NULL, @@ -110,10 +134,10 @@ CREATE TABLE IF NOT EXISTS invoice_htlcs ( resolve_time TIMESTAMP, -- The id of the invoice this htlc belongs to. - invoice_id INTEGER NOT NULL REFERENCES invoices(id), + invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, -- The htlc_id and chan_id identify the htlc. - UNIQUE (htlc_id, chan_id) + UNIQUE (chan_id, htlc_id) ); CREATE INDEX IF NOT EXISTS invoice_htlc_invoice_id_idx ON invoice_htlcs(invoice_id); @@ -129,26 +153,8 @@ CREATE TABLE IF NOT EXISTS invoice_htlc_custom_records ( value BLOB NOT NULL, -- The htlc id this record belongs to. - htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id) + htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS invoice_htlc_custom_records_htlc_id_idx ON invoice_htlc_custom_records(htlc_id); --- invoice_payments contains the information of a settled invoice payment. -CREATE TABLE IF NOT EXISTS invoice_payments ( - -- The id for this invoice payment. Translates to SettleIndex. - id INTEGER PRIMARY KEY, - - -- When the payment was settled. - settled_at TIMESTAMP NOT NULL, - - -- The amount of the payment in millisatoshis. This is the sum of all the - -- the htlcs settled for this payment. - amount_paid_msat BIGINT NOT NULL, - - -- The invoice id this payment is for. - invoice_id INTEGER NOT NULL REFERENCES invoices(id) -); - -CREATE INDEX IF NOT EXISTS invoice_payments_settled_at_idx ON invoice_payments(settled_at); -CREATE INDEX IF NOT EXISTS invoice_payments_invoice_id_idx ON invoice_payments(invoice_id); diff --git a/sqldb/sqlc/migrations/000002_amp_invoices.up.sql b/sqldb/sqlc/migrations/000002_amp_invoices.up.sql index a4daa8c30..7204e775b 100644 --- a/sqldb/sqlc/migrations/000002_amp_invoices.up.sql +++ b/sqldb/sqlc/migrations/000002_amp_invoices.up.sql @@ -1,5 +1,6 @@ --- amp_invoices_payments -CREATE TABLE IF NOT EXISTS amp_invoice_payments ( +-- amp_sub_invoices holds all AMP sub-invoices that belong to a given parent +-- invoice. +CREATE TABLE IF NOT EXISTS amp_sub_invoices ( -- The set id identifying the payment. set_id BLOB PRIMARY KEY, @@ -10,26 +11,35 @@ CREATE TABLE IF NOT EXISTS amp_invoice_payments ( -- Timestamp of when the first htlc for this payment was accepted. created_at TIMESTAMP NOT NULL, - -- If settled, the invoice payment related to this set id. - settled_index INTEGER REFERENCES invoice_payments(id), + -- When the invoice was settled. + settled_at TIMESTAMP, - -- The invoice id this set id is related to. - invoice_id INTEGER NOT NULL REFERENCES invoices(id) + -- If settled, the index is set to the current_value+1 of the settle_index + -- seuqence in the invoice_sequences table. If not settled, then it is set + -- to NULL. + settle_index BIGINT, + + -- The id of the parent invoice this AMP invoice belongs to. + invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, + + -- A unique constraint on the set_id and invoice_id is needed to ensure that + -- we don't have two AMP invoices for the same invoice. + UNIQUE (set_id, invoice_id) ); -CREATE INDEX IF NOT EXISTS amp_invoice_payments_invoice_id_idx ON amp_invoice_payments(invoice_id); +CREATE INDEX IF NOT EXISTS amp_sub_invoices_invoice_id_idx ON amp_sub_invoices(invoice_id); -- amp_invoice_htlcs contains the complementary information for an htlc related -- to an AMP invoice. -CREATE TABLE IF NOT EXISTS amp_invoice_htlcs ( +CREATE TABLE IF NOT EXISTS amp_sub_invoice_htlcs ( + -- The invoice id this entry is related to. + invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, + -- The set id identifying the payment this htlc belongs to. - set_id BLOB NOT NULL REFERENCES amp_invoice_payments(set_id), + set_id BLOB NOT NULL REFERENCES amp_sub_invoices(set_id) ON DELETE CASCADE, -- The id of the htlc this entry blongs to. - htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id), - - -- The invoice id this entry is related to. - invoice_id INTEGER NOT NULL REFERENCES invoices(id), + htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id) ON DELETE CASCADE, -- The root share for this amp htlc. root_share BLOB NOT NULL, @@ -48,7 +58,7 @@ CREATE TABLE IF NOT EXISTS amp_invoice_htlcs ( preimage BLOB ); -CREATE INDEX IF NOT EXISTS amp_htlcs_set_id_idx ON amp_invoice_htlcs(set_id); -CREATE INDEX IF NOT EXISTS amp_htlcs_invoice_id_idx ON amp_invoice_htlcs(invoice_id); -CREATE INDEX IF NOT EXISTS amp_htlcs_htlc_id_idx ON amp_invoice_htlcs(htlc_id); +CREATE INDEX IF NOT EXISTS amp_htlcs_invoice_id_idx ON amp_sub_invoice_htlcs(invoice_id); +CREATE INDEX IF NOT EXISTS amp_htlcs_set_id_idx ON amp_sub_invoice_htlcs(set_id); +CREATE INDEX IF NOT EXISTS amp_htlcs_htlc_id_idx ON amp_sub_invoice_htlcs(htlc_id); diff --git a/sqldb/sqlc/migrations/000003_invoice_events.down.sql b/sqldb/sqlc/migrations/000003_invoice_events.down.sql index 66e554e24..3f447dfa8 100644 --- a/sqldb/sqlc/migrations/000003_invoice_events.down.sql +++ b/sqldb/sqlc/migrations/000003_invoice_events.down.sql @@ -1,9 +1,6 @@ DROP TABLE IF EXISTS invoice_events; - -DROP INDEX IF EXISTS invoice_events_created_at_idx; -DROP INDEX IF EXISTS invoice_events_invoice_id_idx; -DROP INDEX IF EXISTS invoice_events_htlc_id_idx; -DROP INDEX IF EXISTS invoice_events_set_id_idx; +DROP INDEX IF EXISTS invoice_events_added_at_idx; DROP INDEX IF EXISTS invoice_events_event_type_idx; - +DROP INDEX IF EXISTS invoice_events_invoice_id_idx; +DROP INDEX IF EXISTS invoice_events_set_id_idx; DROP TABLE IF EXISTS invoice_event_types; diff --git a/sqldb/sqlc/migrations/000003_invoice_events.up.sql b/sqldb/sqlc/migrations/000003_invoice_events.up.sql index 47d2a91d0..a3718c0eb 100644 --- a/sqldb/sqlc/migrations/000003_invoice_events.up.sql +++ b/sqldb/sqlc/migrations/000003_invoice_events.up.sql @@ -1,59 +1,51 @@ --- invoice_event_types stores the different types of events that can be emitted --- for invoices. +-- invoice_event_types stores the different types of invoice events. CREATE TABLE IF NOT EXISTS invoice_event_types( id INTEGER PRIMARY KEY, description TEXT NOT NULL ); --- invoice_events stores all events related to the node invoices. -CREATE TABLE IF NOT EXISTS invoice_events ( - id INTEGER PRIMARY KEY, - - -- created_at is the creation time of this event. - created_at TIMESTAMP NOT NULL, - - -- invoice_id is the reference to the invoice this event was emitted for. - invoice_id INTEGER NOT NULL REFERENCES invoices(id), - - -- htlc_id is the reference to the htlc this event was emitted for, may be - -- null. - htlc_id BIGINT REFERENCES invoice_htlcs(htlc_id), - - -- set_id is the reference to the set_id this event was emitted for, may be - -- null. - set_id BLOB NOT NULL REFERENCES amp_invoice_payments(set_id), - - -- event_type is the type of this event. - event_type INTEGER NOT NULL REFERENCES invoice_event_types(id), - - -- event_metadata is a TLV encoding any relevant information for this kind - -- of events. - event_metadata BLOB -); - -CREATE INDEX IF NOT EXISTS invoice_events_created_at_idx ON invoice_events(created_at); -CREATE INDEX IF NOT EXISTS invoice_events_invoice_id_idx ON invoice_events(invoice_id); -CREATE INDEX IF NOT EXISTS invoice_events_htlc_id_idx ON invoice_events(htlc_id); -CREATE INDEX IF NOT EXISTS invoice_events_set_id_idx ON invoice_events(set_id); -CREATE INDEX IF NOT EXISTS invoice_events_event_type_idx ON invoice_events(event_type); - - --- invoice_event_types defines the different types of events that can be emitted --- for an invoice. +-- invoice_event_types defines the different types of invoice events. INSERT INTO invoice_event_types (id, description) VALUES - -- invoice_created is the event emitted when an invoice is created. + -- invoice_created is the event type used when an invoice is created. (0, 'invoice_created'), - -- invoice_canceled is the event emitted when an invoice is canceled. - (1, "invoice_canceled"), - -- invoice_settled is the event emitted when an invoice is settled. - (2, "invoice_settled"), - -- setid_created is the event emitted when the first htlc for the set_id is - -- received. - (3, "setid_created"), - -- setid_canceled is the event emitted when the set_id is canceled. - (4, "setid_canceled"), - -- setid_settled is the event emitted when the set_id is settled. - (5, "setid_settled"); + -- invoice_canceled is the event type used when an invoice is canceled. + (1, 'invoice_canceled'), + -- invoice_settled is the event type used when an invoice is settled. + (2, 'invoice_settled'), + -- setid_created is the event type used when an AMP sub invoice + -- corresponding to the set_id is created. + (3, 'setid_created'), + -- setid_canceled is the event type used when an AMP sub invoice + -- corresponding to the set_id is canceled. + (4, 'setid_canceled'), + -- setid_settled is the event type used when an AMP sub invoice + -- corresponding to the set_id is settled. + (5, 'setid_settled'); + +-- invoice_events stores all major events related to the node's invoices and +-- AMP sub invoices. This table can be used to create a historical view of what +-- happened to the node's invoices. +CREATE TABLE IF NOT EXISTS invoice_events ( + id BIGINT PRIMARY KEY, + + -- added_at is the timestamp when this event was added. + added_at TIMESTAMP NOT NULL, + + -- event_type is the type of this event. + event_type INTEGER NOT NULL REFERENCES invoice_event_types(id) ON DELETE CASCADE, + + -- invoice_id is the reference to the invoice this event was added for. + invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, + + -- set_id is the reference to the AMP sub invoice this event was added for. + -- May be NULL if the event is not related to an AMP sub invoice. + set_id BLOB REFERENCES amp_sub_invoices(set_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS invoice_events_added_at_idx ON invoice_events(added_at); +CREATE INDEX IF NOT EXISTS invoice_events_event_type_idx ON invoice_events(event_type); +CREATE INDEX IF NOT EXISTS invoice_events_invoice_id_idx ON invoice_events(invoice_id); +CREATE INDEX IF NOT EXISTS invoice_events_set_id_idx ON invoice_events(set_id); diff --git a/sqldb/sqlc/models.go b/sqldb/sqlc/models.go index 9a72ae2ce..83be5a708 100644 --- a/sqldb/sqlc/models.go +++ b/sqldb/sqlc/models.go @@ -9,50 +9,52 @@ import ( "time" ) -type AmpInvoiceHtlc struct { +type AmpSubInvoice struct { + SetID []byte + State int16 + CreatedAt time.Time + SettledAt sql.NullTime + SettleIndex sql.NullInt64 + InvoiceID int64 +} + +type AmpSubInvoiceHtlc struct { + InvoiceID int64 SetID []byte HtlcID int64 - InvoiceID int32 RootShare []byte ChildIndex int64 Hash []byte Preimage []byte } -type AmpInvoicePayment struct { - SetID []byte - State int16 - CreatedAt time.Time - SettledIndex sql.NullInt32 - InvoiceID int32 -} - type Invoice struct { - ID int32 - Hash []byte - Preimage []byte - Memo sql.NullString - AmountMsat int64 - CltvDelta sql.NullInt32 - Expiry int32 - PaymentAddr []byte - PaymentRequest sql.NullString - State int16 - AmountPaidMsat int64 - IsAmp bool - IsHodl bool - IsKeysend bool - CreatedAt time.Time + ID int64 + Hash []byte + Preimage []byte + SettleIndex sql.NullInt64 + SettledAt sql.NullTime + Memo sql.NullString + AmountMsat int64 + CltvDelta sql.NullInt32 + Expiry int32 + PaymentAddr []byte + PaymentRequest sql.NullString + PaymentRequestHash []byte + State int16 + AmountPaidMsat int64 + IsAmp bool + IsHodl bool + IsKeysend bool + CreatedAt time.Time } type InvoiceEvent struct { - ID int32 - CreatedAt time.Time - InvoiceID int32 - HtlcID sql.NullInt64 - SetID []byte - EventType int32 - EventMetadata []byte + ID int64 + AddedAt time.Time + EventType int32 + InvoiceID int64 + SetID []byte } type InvoiceEventType struct { @@ -62,13 +64,13 @@ type InvoiceEventType struct { type InvoiceFeature struct { Feature int32 - InvoiceID int32 + InvoiceID int64 } type InvoiceHtlc struct { - ID int32 - HtlcID int64 + ID int64 ChanID string + HtlcID int64 AmountMsat int64 TotalMppMsat sql.NullInt64 AcceptHeight int32 @@ -76,7 +78,7 @@ type InvoiceHtlc struct { ExpiryHeight int32 State int16 ResolveTime sql.NullTime - InvoiceID int32 + InvoiceID int64 } type InvoiceHtlcCustomRecord struct { @@ -85,9 +87,7 @@ type InvoiceHtlcCustomRecord struct { HtlcID int64 } -type InvoicePayment struct { - ID int32 - SettledAt time.Time - AmountPaidMsat int64 - InvoiceID int32 +type InvoiceSequence struct { + Name string + CurrentValue int64 } diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index b879f06d3..d55d8090a 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -6,46 +6,43 @@ package sqlc import ( "context" + "database/sql" ) type Querier interface { - DeleteAMPHTLCCustomRecords(ctx context.Context, invoiceID int32) error - DeleteAMPHTLCs(ctx context.Context, invoiceID int32) error - DeleteAMPInvoiceHTLC(ctx context.Context, setID []byte) error - DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) error - DeleteInvoiceEvents(ctx context.Context, invoiceID int32) error - DeleteInvoiceFeatures(ctx context.Context, invoiceID int32) error - DeleteInvoiceHTLC(ctx context.Context, htlcID int64) error - DeleteInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) error - DeleteInvoiceHTLCs(ctx context.Context, invoiceID int32) error - FilterInvoicePayments(ctx context.Context, arg FilterInvoicePaymentsParams) ([]FilterInvoicePaymentsRow, error) + DeleteCanceledInvoices(ctx context.Context) (sql.Result, error) + DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) (sql.Result, error) + FetchAMPSubInvoiceHTLCs(ctx context.Context, arg FetchAMPSubInvoiceHTLCsParams) ([]FetchAMPSubInvoiceHTLCsRow, error) + FetchAMPSubInvoices(ctx context.Context, arg FetchAMPSubInvoicesParams) ([]AmpSubInvoice, error) + FetchSettledAMPSubInvoices(ctx context.Context, arg FetchSettledAMPSubInvoicesParams) ([]FetchSettledAMPSubInvoicesRow, error) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) ([]Invoice, error) - GetAMPInvoiceHTLCsByInvoiceID(ctx context.Context, invoiceID int32) ([]AmpInvoiceHtlc, error) - GetAMPInvoiceHTLCsBySetID(ctx context.Context, setID []byte) ([]AmpInvoiceHtlc, error) + GetAMPInvoiceID(ctx context.Context, setID []byte) (int64, error) // This method may return more than one invoice if filter using multiple fields // from different invoices. It is the caller's responsibility to ensure that // we bubble up an error in those cases. GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoice, error) - GetInvoiceFeatures(ctx context.Context, invoiceID int32) ([]InvoiceFeature, error) - GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) ([]GetInvoiceHTLCCustomRecordsRow, error) - GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]InvoiceHtlc, error) - GetInvoicePayments(ctx context.Context, invoiceID int32) ([]InvoicePayment, error) - GetSetIDHTLCsCustomRecords(ctx context.Context, setID []byte) ([]GetSetIDHTLCsCustomRecordsRow, error) - InsertAMPInvoiceHTLC(ctx context.Context, arg InsertAMPInvoiceHTLCParams) error - InsertAMPInvoicePayment(ctx context.Context, arg InsertAMPInvoicePaymentParams) error - InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int32, error) - InsertInvoiceEvent(ctx context.Context, arg InsertInvoiceEventParams) error + GetInvoiceFeatures(ctx context.Context, invoiceID int64) ([]InvoiceFeature, error) + GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error) + GetInvoiceHTLCs(ctx context.Context, invoiceID int64) ([]InvoiceHtlc, error) + InsertAMPSubInvoiceHTLC(ctx context.Context, arg InsertAMPSubInvoiceHTLCParams) error + InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int64, error) InsertInvoiceFeature(ctx context.Context, arg InsertInvoiceFeatureParams) error - InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) error + InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) (int64, error) InsertInvoiceHTLCCustomRecord(ctx context.Context, arg InsertInvoiceHTLCCustomRecordParams) error - InsertInvoicePayment(ctx context.Context, arg InsertInvoicePaymentParams) (int32, error) - SelectAMPInvoicePayments(ctx context.Context, arg SelectAMPInvoicePaymentsParams) ([]SelectAMPInvoicePaymentsRow, error) - SelectInvoiceEvents(ctx context.Context, arg SelectInvoiceEventsParams) ([]InvoiceEvent, error) - UpdateAMPInvoiceHTLC(ctx context.Context, arg UpdateAMPInvoiceHTLCParams) error - UpdateAMPPayment(ctx context.Context, arg UpdateAMPPaymentParams) error - UpdateInvoice(ctx context.Context, arg UpdateInvoiceParams) error + NextInvoiceSettleIndex(ctx context.Context) (int64, error) + OnAMPSubInvoiceCanceled(ctx context.Context, arg OnAMPSubInvoiceCanceledParams) error + OnAMPSubInvoiceCreated(ctx context.Context, arg OnAMPSubInvoiceCreatedParams) error + OnAMPSubInvoiceSettled(ctx context.Context, arg OnAMPSubInvoiceSettledParams) error + OnInvoiceCanceled(ctx context.Context, arg OnInvoiceCanceledParams) error + OnInvoiceCreated(ctx context.Context, arg OnInvoiceCreatedParams) error + OnInvoiceSettled(ctx context.Context, arg OnInvoiceSettledParams) error + UpdateAMPSubInvoiceHTLCPreimage(ctx context.Context, arg UpdateAMPSubInvoiceHTLCPreimageParams) (sql.Result, error) + UpdateAMPSubInvoiceState(ctx context.Context, arg UpdateAMPSubInvoiceStateParams) error + UpdateInvoiceAmountPaid(ctx context.Context, arg UpdateInvoiceAmountPaidParams) (sql.Result, error) UpdateInvoiceHTLC(ctx context.Context, arg UpdateInvoiceHTLCParams) error UpdateInvoiceHTLCs(ctx context.Context, arg UpdateInvoiceHTLCsParams) error + UpdateInvoiceState(ctx context.Context, arg UpdateInvoiceStateParams) (sql.Result, error) + UpsertAMPSubInvoice(ctx context.Context, arg UpsertAMPSubInvoiceParams) (sql.Result, error) } var _ Querier = (*Queries)(nil) diff --git a/sqldb/sqlc/queries/amp_invoices.sql b/sqldb/sqlc/queries/amp_invoices.sql index d965de768..3b6ee76ac 100644 --- a/sqldb/sqlc/queries/amp_invoices.sql +++ b/sqldb/sqlc/queries/amp_invoices.sql @@ -1,79 +1,67 @@ --- name: InsertAMPInvoicePayment :exec -INSERT INTO amp_invoice_payments ( - set_id, state, created_at, settled_index, invoice_id +-- name: UpsertAMPSubInvoice :execresult +INSERT INTO amp_sub_invoices ( + set_id, state, created_at, invoice_id ) VALUES ( - $1, $2, $3, $4, $5 -); + $1, $2, $3, $4 +) ON CONFLICT (set_id, invoice_id) DO NOTHING; --- name: SelectAMPInvoicePayments :many -SELECT aip.*, ip.* -FROM amp_invoice_payments aip LEFT JOIN invoice_payments ip ON aip.settled_index = ip.id -WHERE ( - set_id = sqlc.narg('set_id') OR - sqlc.narg('set_id') IS NULL -) AND ( - aip.settled_index = sqlc.narg('settled_index') OR - sqlc.narg('settled_index') IS NULL -) AND ( - aip.invoice_id = sqlc.narg('invoice_id') OR - sqlc.narg('invoice_id') IS NULL -); +-- name: GetAMPInvoiceID :one +SELECT invoice_id FROM amp_sub_invoices WHERE set_id = $1; --- name: UpdateAMPPayment :exec -UPDATE amp_invoice_payments -SET state = $1, settled_index = $2 -WHERE state = 0 AND ( - set_id = sqlc.narg('set_id') OR - sqlc.narg('set_id') IS NULL -) AND ( - invoice_id = sqlc.narg('invoice_id') OR - sqlc.narg('invoice_id') IS NULL -); +-- name: UpdateAMPSubInvoiceState :exec +UPDATE amp_sub_invoices +SET state = $2, + settle_index = COALESCE(settle_index, $3), + settled_at = COALESCE(settled_at, $4) +WHERE set_id = $1; --- name: InsertAMPInvoiceHTLC :exec -INSERT INTO amp_invoice_htlcs ( - set_id, htlc_id, root_share, child_index, hash, preimage +-- name: InsertAMPSubInvoiceHTLC :exec +INSERT INTO amp_sub_invoice_htlcs ( + invoice_id, set_id, htlc_id, root_share, child_index, hash, preimage ) VALUES ( - $1, $2, $3, $4, $5, $6 + $1, $2, $3, $4, $5, $6, $7 ); --- name: GetAMPInvoiceHTLCsBySetID :many +-- name: FetchAMPSubInvoices :many SELECT * -FROM amp_invoice_htlcs -WHERE set_id = $1; +FROM amp_sub_invoices +WHERE invoice_id = $1 +AND ( + set_id = sqlc.narg('set_id') OR + sqlc.narg('set_id') IS NULL +); --- name: GetAMPInvoiceHTLCsByInvoiceID :many -SELECT * -FROM amp_invoice_htlcs -WHERE invoice_id = $1; +-- name: FetchAMPSubInvoiceHTLCs :many +SELECT + amp.set_id, amp.root_share, amp.child_index, amp.hash, amp.preimage, + invoice_htlcs.* +FROM amp_sub_invoice_htlcs amp +INNER JOIN invoice_htlcs ON amp.htlc_id = invoice_htlcs.id +WHERE amp.invoice_id = $1 +AND ( + set_id = sqlc.narg('set_id') OR + sqlc.narg('set_id') IS NULL +); --- name: GetSetIDHTLCsCustomRecords :many -SELECT ihcr.htlc_id, key, value -FROM amp_invoice_htlcs aih JOIN invoice_htlc_custom_records ihcr ON aih.id=ihcr.htlc_id -WHERE aih.set_id = $1; - --- name: UpdateAMPInvoiceHTLC :exec -UPDATE amp_invoice_htlcs -SET preimage = $1 -WHERE htlc_id = $2; - --- name: DeleteAMPHTLCCustomRecords :exec -WITH htlc_ids AS ( - SELECT htlc_id - FROM amp_invoice_htlcs - WHERE invoice_id = $1 -) -DELETE -FROM invoice_htlc_custom_records -WHERE htlc_id IN (SELECT id FROM htlc_ids); - --- name: DeleteAMPHTLCs :exec -DELETE -FROM amp_invoice_htlcs -WHERE invoice_id = $1; - --- name: DeleteAMPInvoiceHTLC :exec -DELETE -FROM amp_invoice_htlcs -WHERE set_id = $1; +-- name: FetchSettledAMPSubInvoices :many +SELECT + a.set_id, + a.settle_index as amp_settle_index, + a.settled_at as amp_settled_at, + i.* +FROM amp_sub_invoices a +INNER JOIN invoices i ON a.invoice_id = i.id +WHERE ( + a.settle_index >= sqlc.narg('settle_index_get') OR + sqlc.narg('settle_index_get') IS NULL +) AND ( + a.settle_index <= sqlc.narg('settle_index_let') OR + sqlc.narg('settle_index_let') IS NULL +); +-- name: UpdateAMPSubInvoiceHTLCPreimage :execresult +UPDATE amp_sub_invoice_htlcs AS a +SET preimage = $4 +WHERE a.invoice_id = $1 AND a.set_id = $2 AND a.htlc_id = ( + SELECT id FROM invoice_htlcs AS i WHERE i.htlc_id = $3 +); diff --git a/sqldb/sqlc/queries/invoice_events.sql b/sqldb/sqlc/queries/invoice_events.sql index 4216a2513..4192c466b 100644 --- a/sqldb/sqlc/queries/invoice_events.sql +++ b/sqldb/sqlc/queries/invoice_events.sql @@ -1,35 +1,41 @@ --- name: InsertInvoiceEvent :exec +-- name: OnInvoiceCreated :exec INSERT INTO invoice_events ( - created_at, invoice_id, htlc_id, set_id, event_type, event_metadata + added_at, event_type, invoice_id ) VALUES ( - $1, $2, $3, $4, $5, $6 + $1, 0, $2 ); --- name: SelectInvoiceEvents :many -SELECT * -FROM invoice_events -WHERE ( - invoice_id = sqlc.narg('invoice_id') OR - sqlc.narg('invoice_id') IS NULL -) AND ( - htlc_id = sqlc.narg('htlc_id') OR - sqlc.narg('htlc_id') IS NULL -) AND ( - set_id = sqlc.narg('set_id') OR - sqlc.narg('set_id') IS NULL -) AND ( - event_type = sqlc.narg('event_type') OR - sqlc.narg('event_type') IS NULL -) AND ( - created_at >= sqlc.narg('created_after') OR - sqlc.narg('created_after') IS NULL -) AND ( - created_at <= sqlc.narg('created_before') OR - sqlc.narg('created_before') IS NULL -) -LIMIT @num_limit OFFSET @num_offset; +-- name: OnInvoiceCanceled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id +) VALUES ( + $1, 1, $2 +); --- name: DeleteInvoiceEvents :exec -DELETE -FROM invoice_events -WHERE invoice_id = $1; +-- name: OnInvoiceSettled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id +) VALUES ( + $1, 2, $2 +); + +-- name: OnAMPSubInvoiceCreated :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id, set_id +) VALUES ( + $1, 3, $2, $3 +); + +-- name: OnAMPSubInvoiceCanceled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id, set_id +) VALUES ( + $1, 4, $2, $3 +); + +-- name: OnAMPSubInvoiceSettled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id, set_id +) VALUES ( + $1, 5, $2, $3 +); diff --git a/sqldb/sqlc/queries/invoices.sql b/sqldb/sqlc/queries/invoices.sql index 335929e05..c36976a25 100644 --- a/sqldb/sqlc/queries/invoices.sql +++ b/sqldb/sqlc/queries/invoices.sql @@ -1,10 +1,10 @@ -- name: InsertInvoice :one INSERT INTO invoices ( hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, - payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, - created_at + payment_request, payment_request_hash, state, amount_paid_msat, is_amp, + is_hodl, is_keysend, created_at ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15 ) RETURNING id; -- name: InsertInvoiceFeature :exec @@ -19,35 +19,36 @@ SELECT * FROM invoice_features WHERE invoice_id = $1; --- name: DeleteInvoiceFeatures :exec -DELETE -FROM invoice_features -WHERE invoice_id = $1; - -- This method may return more than one invoice if filter using multiple fields -- from different invoices. It is the caller's responsibility to ensure that -- we bubble up an error in those cases. -- name: GetInvoice :many -SELECT * -FROM invoices +SELECT i.* +FROM invoices i +LEFT JOIN amp_sub_invoices a on i.id = a.invoice_id WHERE ( - id = sqlc.narg('add_index') OR + i.id = sqlc.narg('add_index') OR sqlc.narg('add_index') IS NULL ) AND ( - hash = sqlc.narg('hash') OR + i.hash = sqlc.narg('hash') OR sqlc.narg('hash') IS NULL ) AND ( - preimage = sqlc.narg('preimage') OR + i.preimage = sqlc.narg('preimage') OR sqlc.narg('preimage') IS NULL ) AND ( - payment_addr = sqlc.narg('payment_addr') OR + i.payment_addr = sqlc.narg('payment_addr') OR sqlc.narg('payment_addr') IS NULL +) AND ( + a.set_id = sqlc.narg('set_id') OR + sqlc.narg('set_id') IS NULL ) +GROUP BY i.id LIMIT 2; -- name: FilterInvoices :many -SELECT * +SELECT + invoices.* FROM invoices WHERE ( id >= sqlc.narg('add_index_get') OR @@ -55,6 +56,12 @@ WHERE ( ) AND ( id <= sqlc.narg('add_index_let') OR sqlc.narg('add_index_let') IS NULL +) AND ( + settle_index >= sqlc.narg('settle_index_get') OR + sqlc.narg('settle_index_get') IS NULL +) AND ( + settle_index <= sqlc.narg('settle_index_let') OR + sqlc.narg('settle_index_let') IS NULL ) AND ( state = sqlc.narg('state') OR sqlc.narg('state') IS NULL @@ -81,12 +88,25 @@ ORDER BY END DESC LIMIT @num_limit OFFSET @num_offset; --- name: UpdateInvoice :exec -UPDATE invoices -SET preimage=$2, state=$3, amount_paid_msat=$4 -WHERE id=$1; +-- name: UpdateInvoiceState :execresult +UPDATE invoices +SET state = $2, + preimage = COALESCE(preimage, $3), + settle_index = COALESCE(settle_index, $4), + settled_at = COALESCE(settled_at, $5) +WHERE id = $1; --- name: DeleteInvoice :exec +-- name: UpdateInvoiceAmountPaid :execresult +UPDATE invoices +SET amount_paid_msat = $2 +WHERE id = $1; + +-- name: NextInvoiceSettleIndex :one +UPDATE invoice_sequences SET current_value = current_value + 1 +WHERE name = 'settle_index' +RETURNING current_value; + +-- name: DeleteInvoice :execresult DELETE FROM invoices WHERE ( @@ -96,20 +116,25 @@ WHERE ( hash = sqlc.narg('hash') OR sqlc.narg('hash') IS NULL ) AND ( - preimage = sqlc.narg('preimage') OR - sqlc.narg('preimage') IS NULL + settle_index = sqlc.narg('settle_index') OR + sqlc.narg('settle_index') IS NULL ) AND ( payment_addr = sqlc.narg('payment_addr') OR sqlc.narg('payment_addr') IS NULL ); --- name: InsertInvoiceHTLC :exec +-- name: DeleteCanceledInvoices :execresult +DELETE +FROM invoices +WHERE state = 2; + +-- name: InsertInvoiceHTLC :one INSERT INTO invoice_htlcs ( htlc_id, chan_id, amount_msat, total_mpp_msat, accept_height, accept_time, expiry_height, state, resolve_time, invoice_id ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 -); +) RETURNING id; -- name: GetInvoiceHTLCs :many SELECT * @@ -118,25 +143,14 @@ WHERE invoice_id = $1; -- name: UpdateInvoiceHTLC :exec UPDATE invoice_htlcs -SET state=$2, resolve_time=$3 -WHERE id = $1; +SET state=$4, resolve_time=$5 +WHERE htlc_id = $1 AND chan_id = $2 AND invoice_id = $3; -- name: UpdateInvoiceHTLCs :exec UPDATE invoice_htlcs SET state=$2, resolve_time=$3 WHERE invoice_id = $1 AND resolve_time IS NULL; - --- name: DeleteInvoiceHTLC :exec -DELETE -FROM invoice_htlcs -WHERE htlc_id = $1; - --- name: DeleteInvoiceHTLCs :exec -DELETE -FROM invoice_htlcs -WHERE invoice_id = $1; - -- name: InsertInvoiceHTLCCustomRecord :exec INSERT INTO invoice_htlc_custom_records ( key, value, htlc_id @@ -148,49 +162,3 @@ INSERT INTO invoice_htlc_custom_records ( SELECT ihcr.htlc_id, key, value FROM invoice_htlcs ih JOIN invoice_htlc_custom_records ihcr ON ih.id=ihcr.htlc_id WHERE ih.invoice_id = $1; - --- name: DeleteInvoiceHTLCCustomRecords :exec -WITH htlc_ids AS ( - SELECT ih.id - FROM invoice_htlcs ih JOIN invoice_htlc_custom_records ihcr ON ih.id=ihcr.htlc_id - WHERE ih.invoice_id = $1 -) -DELETE -FROM invoice_htlc_custom_records -WHERE htlc_id IN (SELECT id FROM htlc_ids); - --- name: InsertInvoicePayment :one -INSERT INTO invoice_payments ( - invoice_id, amount_paid_msat, settled_at -) VALUES ( - $1, $2, $3 -) RETURNING id; - --- name: GetInvoicePayments :many -SELECT * -FROM invoice_payments -WHERE invoice_id = $1; - --- name: FilterInvoicePayments :many -SELECT - ip.id AS settle_index, ip.amount_paid_msat, ip.settled_at AS settle_date, - i.* -FROM invoice_payments ip JOIN invoices i ON ip.invoice_id = i.id -WHERE ( - ip.id >= sqlc.narg('settle_index_get') OR - sqlc.narg('settle_index_get') IS NULL -) AND ( - ip.settled_at >= sqlc.narg('settled_after') OR - sqlc.narg('settled_after') IS NULL -) -ORDER BY - CASE - WHEN sqlc.narg('reverse') = FALSE THEN ip.id - ELSE NULL - END ASC, - CASE - WHEN sqlc.narg('reverse') = TRUE THEN ip.id - ELSE NULL - END DESC -LIMIT @num_limit OFFSET @num_offset; - From 74e7a50f4a2d5f6f97c0f21aa69e2d1d502a853a Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 22 Nov 2023 16:47:00 +0100 Subject: [PATCH 04/21] sqldb: convert primary key violation to ErrSQLUniqueConstraintViolation Besides the usual unique constraint violation that we already support we also want to return the same error if the primary key constraint is violated. --- sqldb/sqlerrors.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sqldb/sqlerrors.go b/sqldb/sqlerrors.go index c595bd528..930347676 100644 --- a/sqldb/sqlerrors.go +++ b/sqldb/sqlerrors.go @@ -46,6 +46,11 @@ func parseSqliteError(sqliteErr *sqlite.Error) error { DBError: sqliteErr, } + case sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: + return &ErrSQLUniqueConstraintViolation{ + DBError: sqliteErr, + } + // Database is currently busy, so we'll need to try again. case sqlite3.SQLITE_BUSY: return &ErrSerializationError{ From 5cf67c444088882e4c4834f85df481a6ac5b301a Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 22 Nov 2023 16:49:20 +0100 Subject: [PATCH 05/21] sqldb: add utility functions for SQL NULL types --- sqldb/sqlutils.go | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 sqldb/sqlutils.go diff --git a/sqldb/sqlutils.go b/sqldb/sqlutils.go new file mode 100644 index 000000000..b71c5f00c --- /dev/null +++ b/sqldb/sqlutils.go @@ -0,0 +1,54 @@ +package sqldb + +import ( + "database/sql" + "time" + + "golang.org/x/exp/constraints" +) + +// sqlInt32 turns a numerical integer type into the NullInt32 that sql/sqlc +// uses when an integer field can be permitted to be NULL. +// +// We use this constraints.Integer constraint here which maps to all signed and +// unsigned integer types. +func sqlInt32[T constraints.Integer](num T) sql.NullInt32 { + return sql.NullInt32{ + Int32: int32(num), + Valid: true, + } +} + +// sqlInt64 turns a numerical integer type into the NullInt64 that sql/sqlc +// uses when an integer field can be permitted to be NULL. +// +// We use this constraints.Integer constraint here which maps to all signed and +// unsigned integer types. +func sqlInt64[T constraints.Integer](num T) sql.NullInt64 { + return sql.NullInt64{ + Int64: int64(num), + Valid: true, + } +} + +// sqlStr turns a string into the NullString that sql/sqlc uses when a string +// can be permitted to be NULL. +func sqlStr(s string) sql.NullString { + if s == "" { + return sql.NullString{} + } + + return sql.NullString{ + String: s, + Valid: true, + } +} + +// sqlTime turns a time.Time into the NullTime that sql/sqlc uses when a time +// can be permitted to be NULL. +func sqlTime(t time.Time) sql.NullTime { + return sql.NullTime{ + Time: t, + Valid: true, + } +} From 3d56d2f1ee0ab1a0bc901e64c6c14d53fcc68695 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 22 Nov 2023 16:50:06 +0100 Subject: [PATCH 06/21] sqldb: add full SQL InvoiceStore implementation --- sqldb/invoices.go | 1658 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1658 insertions(+) create mode 100644 sqldb/invoices.go diff --git a/sqldb/invoices.go b/sqldb/invoices.go new file mode 100644 index 000000000..6e2cb47a1 --- /dev/null +++ b/sqldb/invoices.go @@ -0,0 +1,1658 @@ +package sqldb + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "math" + "strconv" + "time" + + "github.com/lightningnetwork/lnd/channeldb/models" + "github.com/lightningnetwork/lnd/clock" + invpkg "github.com/lightningnetwork/lnd/invoices" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/sqldb/sqlc" +) + +const ( + // queryPaginationLimit is used in the LIMIT clause of the SQL queries + // to limit the number of rows returned. + queryPaginationLimit = 100 +) + +// InvoiceQueries is an interface that defines the set of operations that can be +// executed against the invoice database. +type InvoiceQueries interface { //nolint:interfacebloat + InsertInvoice(ctx context.Context, arg sqlc.InsertInvoiceParams) (int64, + error) + + InsertInvoiceFeature(ctx context.Context, + arg sqlc.InsertInvoiceFeatureParams) error + + InsertInvoiceHTLC(ctx context.Context, + arg sqlc.InsertInvoiceHTLCParams) (int64, error) + + InsertInvoiceHTLCCustomRecord(ctx context.Context, + arg sqlc.InsertInvoiceHTLCCustomRecordParams) error + + FilterInvoices(ctx context.Context, + arg sqlc.FilterInvoicesParams) ([]sqlc.Invoice, error) + + GetInvoice(ctx context.Context, + arg sqlc.GetInvoiceParams) ([]sqlc.Invoice, error) + + GetInvoiceFeatures(ctx context.Context, + invoiceID int64) ([]sqlc.InvoiceFeature, error) + + GetInvoiceHTLCCustomRecords(ctx context.Context, + invoiceID int64) ([]sqlc.GetInvoiceHTLCCustomRecordsRow, error) + + GetInvoiceHTLCs(ctx context.Context, + invoiceID int64) ([]sqlc.InvoiceHtlc, error) + + UpdateInvoiceState(ctx context.Context, + arg sqlc.UpdateInvoiceStateParams) (sql.Result, error) + + UpdateInvoiceAmountPaid(ctx context.Context, + arg sqlc.UpdateInvoiceAmountPaidParams) (sql.Result, error) + + NextInvoiceSettleIndex(ctx context.Context) (int64, error) + + UpdateInvoiceHTLC(ctx context.Context, + arg sqlc.UpdateInvoiceHTLCParams) error + + DeleteInvoice(ctx context.Context, arg sqlc.DeleteInvoiceParams) ( + sql.Result, error) + + DeleteCanceledInvoices(ctx context.Context) (sql.Result, error) + + // AMP sub invoice specific methods. + UpsertAMPSubInvoice(ctx context.Context, + arg sqlc.UpsertAMPSubInvoiceParams) (sql.Result, error) + + UpdateAMPSubInvoiceState(ctx context.Context, + arg sqlc.UpdateAMPSubInvoiceStateParams) error + + InsertAMPSubInvoiceHTLC(ctx context.Context, + arg sqlc.InsertAMPSubInvoiceHTLCParams) error + + FetchAMPSubInvoices(ctx context.Context, + arg sqlc.FetchAMPSubInvoicesParams) ([]sqlc.AmpSubInvoice, + error) + + FetchAMPSubInvoiceHTLCs(ctx context.Context, + arg sqlc.FetchAMPSubInvoiceHTLCsParams) ( + []sqlc.FetchAMPSubInvoiceHTLCsRow, error) + + FetchSettledAMPSubInvoices(ctx context.Context, + arg sqlc.FetchSettledAMPSubInvoicesParams) ( + []sqlc.FetchSettledAMPSubInvoicesRow, error) + + UpdateAMPSubInvoiceHTLCPreimage(ctx context.Context, + arg sqlc.UpdateAMPSubInvoiceHTLCPreimageParams) (sql.Result, + error) + + // Invoice events specific methods. + OnInvoiceCreated(ctx context.Context, + arg sqlc.OnInvoiceCreatedParams) error + + OnInvoiceCanceled(ctx context.Context, + arg sqlc.OnInvoiceCanceledParams) error + + OnInvoiceSettled(ctx context.Context, + arg sqlc.OnInvoiceSettledParams) error + + OnAMPSubInvoiceCreated(ctx context.Context, + arg sqlc.OnAMPSubInvoiceCreatedParams) error + + OnAMPSubInvoiceCanceled(ctx context.Context, + arg sqlc.OnAMPSubInvoiceCanceledParams) error + + OnAMPSubInvoiceSettled(ctx context.Context, + arg sqlc.OnAMPSubInvoiceSettledParams) error +} + +var _ invpkg.InvoiceDB = (*InvoiceStore)(nil) + +// InvoiceQueriesTxOptions defines the set of db txn options the InvoiceQueries +// understands. +type InvoiceQueriesTxOptions struct { + // readOnly governs if a read only transaction is needed or not. + readOnly bool +} + +// ReadOnly returns true if the transaction should be read only. +// +// NOTE: This implements the TxOptions. +func (a *InvoiceQueriesTxOptions) ReadOnly() bool { + return a.readOnly +} + +// NewInvoiceQueryReadTx creates a new read transaction option set. +func NewInvoiceQueryReadTx() InvoiceQueriesTxOptions { + return InvoiceQueriesTxOptions{ + readOnly: true, + } +} + +// BatchedInvoiceQueries is a version of the InvoiceQueries that's capable of +// batched database operations. +type BatchedInvoiceQueries interface { + InvoiceQueries + + BatchedTx[InvoiceQueries] +} + +// InvoiceStore represents a storage backend. +type InvoiceStore struct { + db BatchedInvoiceQueries + clock clock.Clock +} + +// NewInvoiceStore creates a new InvoiceStore instance given a open +// BatchedInvoiceQueries storage backend. +func NewInvoiceStore(db BatchedInvoiceQueries, + clock clock.Clock) *InvoiceStore { + + return &InvoiceStore{ + db: db, + clock: clock, + } +} + +// AddInvoice inserts the targeted invoice into the database. If the invoice has +// *any* payment hashes which already exists within the database, then the +// insertion will be aborted and rejected due to the strict policy banning any +// duplicate payment hashes. +// +// NOTE: A side effect of this function is that it sets AddIndex on newInvoice. +func (i *InvoiceStore) AddInvoice(ctx context.Context, + newInvoice *invpkg.Invoice, paymentHash lntypes.Hash) (uint64, error) { + + // Make sure this is a valid invoice before trying to store it in our + // DB. + if err := invpkg.ValidateInvoice(newInvoice, paymentHash); err != nil { + return 0, err + } + + var ( + writeTxOpts InvoiceQueriesTxOptions + invoiceID int64 + ) + + // Precompute the payment request hash so we can use it in the query. + var paymentRequestHash []byte + if len(newInvoice.PaymentRequest) > 0 { + h := sha256.New() + h.Write(newInvoice.PaymentRequest) + paymentRequestHash = h.Sum(nil) + } + + err := i.db.ExecTx(ctx, &writeTxOpts, func(db InvoiceQueries) error { + params := sqlc.InsertInvoiceParams{ + Hash: paymentHash[:], + Memo: sqlStr(string(newInvoice.Memo)), + AmountMsat: int64(newInvoice.Terms.Value), + // Note: BOLT12 invoices don't have a final cltv delta. + CltvDelta: sqlInt32(newInvoice.Terms.FinalCltvDelta), + Expiry: int32(newInvoice.Terms.Expiry), + // Note: keysend invoices don't have a payment request. + PaymentRequest: sqlStr(string( + newInvoice.PaymentRequest), + ), + PaymentRequestHash: paymentRequestHash, + State: int16(newInvoice.State), + AmountPaidMsat: int64(newInvoice.AmtPaid), + IsAmp: newInvoice.IsAMP(), + IsHodl: newInvoice.HodlInvoice, + IsKeysend: newInvoice.IsKeysend(), + CreatedAt: newInvoice.CreationDate.UTC(), + } + + // Some invoices may not have a preimage, like in the case of + // HODL invoices. + if newInvoice.Terms.PaymentPreimage != nil { + preimage := *newInvoice.Terms.PaymentPreimage + if preimage == invpkg.UnknownPreimage { + return errors.New("cannot use all-zeroes " + + "preimage") + } + params.Preimage = preimage[:] + } + + // Some non MPP payments may have the default (invalid) value. + if newInvoice.Terms.PaymentAddr != invpkg.BlankPayAddr { + params.PaymentAddr = newInvoice.Terms.PaymentAddr[:] + } + + var err error + invoiceID, err = db.InsertInvoice(ctx, params) + if err != nil { + return fmt.Errorf("unable to insert invoice: %w", err) + } + + // TODO(positiveblue): if invocies do not have custom features + // maybe just store the "invoice type" and populate the features + // based on that. + for feature := range newInvoice.Terms.Features.Features() { + params := sqlc.InsertInvoiceFeatureParams{ + InvoiceID: invoiceID, + Feature: int32(feature), + } + + err := db.InsertInvoiceFeature(ctx, params) + if err != nil { + return fmt.Errorf("unable to insert invoice "+ + "feature(%v): %w", feature, err) + } + } + + // Finally add a new event for this invoice. + return db.OnInvoiceCreated(ctx, sqlc.OnInvoiceCreatedParams{ + AddedAt: newInvoice.CreationDate.UTC(), + InvoiceID: invoiceID, + }) + }) + if err != nil { + mappedSQLErr := MapSQLError(err) + var uniqueConstraintErr *ErrSQLUniqueConstraintViolation + if errors.As(mappedSQLErr, &uniqueConstraintErr) { + // Add context to unique constraint errors. + return 0, invpkg.ErrDuplicateInvoice + } + + return 0, fmt.Errorf("unable to add invoice(%v): %w", + paymentHash, err) + } + + newInvoice.AddIndex = uint64(invoiceID) + + return newInvoice.AddIndex, nil +} + +// fetchInvoice fetches the common invoice data and the AMP state for the +// invoice with the given reference. +func (i *InvoiceStore) fetchInvoice(ctx context.Context, + db InvoiceQueries, ref invpkg.InvoiceRef) (*invpkg.Invoice, error) { + + if ref.PayHash() == nil && ref.PayAddr() == nil && ref.SetID() == nil { + return nil, invpkg.ErrInvoiceNotFound + } + + var ( + invoice *invpkg.Invoice + params sqlc.GetInvoiceParams + ) + + // Given all invoices are uniquely identified by their payment hash, + // we can use it to query a specific invoice. + if ref.PayHash() != nil { + params.Hash = ref.PayHash()[:] + } + + // Newer invoices (0.11 and up) are indexed by payment address in + // addition to payment hash, but pre 0.8 invoices do not have one at + // all. Only allow lookups for payment address if it is not a blank + // payment address, which is a special-cased value for legacy keysend + // invoices. + if ref.PayAddr() != nil && *ref.PayAddr() != invpkg.BlankPayAddr { + params.PaymentAddr = ref.PayAddr()[:] + } + + // If the reference has a set ID we'll fetch the invoice which has the + // corresponding AMP sub invoice. + if ref.SetID() != nil { + params.SetID = ref.SetID()[:] + } + + rows, err := db.GetInvoice(ctx, params) + switch { + case len(rows) == 0: + return nil, invpkg.ErrInvoiceNotFound + + case len(rows) > 1: + // In case the reference is ambiguous, meaning it matches more + // than one invoice, we'll return an error. + return nil, fmt.Errorf("ambiguous invoice ref: %s", + ref.String()) + + case err != nil: + return nil, fmt.Errorf("unable to fetch invoice: %w", err) + } + + var ( + setID *[32]byte + fetchAmpHtlcs bool + ) + + // Now that we got the invoice itself, fetch the HTLCs as requested by + // the modifier. + switch ref.Modifier() { + case invpkg.DefaultModifier: + // By default we'll fetch all AMP HTLCs. + setID = nil + fetchAmpHtlcs = true + + case invpkg.HtlcSetOnlyModifier: + // In this case we'll fetch all AMP HTLCs for the + // specified set id. + if ref.SetID() == nil { + return nil, fmt.Errorf("set ID is required to use " + + "the HTLC set only modifier") + } + + setID = ref.SetID() + fetchAmpHtlcs = true + + case invpkg.HtlcSetBlankModifier: + // No need to fetch any HTLCs. + setID = nil + fetchAmpHtlcs = false + + default: + return nil, fmt.Errorf("unknown invoice ref modifier: %v", + ref.Modifier()) + } + + // Fetch the rest of the invoice data and fill the invoice struct. + _, invoice, err = fetchInvoiceData( + ctx, db, rows[0], setID, fetchAmpHtlcs, + ) + if err != nil { + return nil, err + } + + return invoice, nil +} + +// fetchAmpState fetches the AMP state for the invoice with the given ID. +// Optional setID can be provided to fetch the state for a specific AMP HTLC +// set. If setID is nil then we'll fetch the state for all AMP sub invoices. If +// fetchHtlcs is set to true, the HTLCs for the given set will be fetched as +// well. +// +//nolint:funlen +func fetchAmpState(ctx context.Context, db InvoiceQueries, invoiceID int64, + setID *[32]byte, fetchHtlcs bool) (invpkg.AMPInvoiceState, + invpkg.HTLCSet, error) { + + var paramSetID []byte + if setID != nil { + paramSetID = setID[:] + } + + // First fetch all the AMP sub invoices for this invoice or the one + // matching the provided set ID. + ampInvoiceRows, err := db.FetchAMPSubInvoices( + ctx, sqlc.FetchAMPSubInvoicesParams{ + InvoiceID: invoiceID, + SetID: paramSetID, + }, + ) + if err != nil { + return nil, nil, err + } + + ampState := make(map[invpkg.SetID]invpkg.InvoiceStateAMP) + for _, row := range ampInvoiceRows { + var rowSetID [32]byte + + if len(row.SetID) != 32 { + return nil, nil, fmt.Errorf("invalid set id length: %d", + len(row.SetID)) + } + + var settleDate time.Time + if row.SettledAt.Valid { + settleDate = row.SettledAt.Time.Local() + } + + copy(rowSetID[:], row.SetID) + ampState[rowSetID] = invpkg.InvoiceStateAMP{ + State: invpkg.HtlcState(row.State), + SettleIndex: uint64(row.SettleIndex.Int64), + SettleDate: settleDate, + InvoiceKeys: make(map[models.CircuitKey]struct{}), + } + } + + if !fetchHtlcs { + return ampState, nil, nil + } + + customRecordRows, err := db.GetInvoiceHTLCCustomRecords(ctx, invoiceID) + if err != nil { + return nil, nil, fmt.Errorf("unable to get custom records for "+ + "invoice HTLCs: %w", err) + } + + customRecords := make(map[int64]record.CustomSet, len(customRecordRows)) + for _, row := range customRecordRows { + if _, ok := customRecords[row.HtlcID]; !ok { + customRecords[row.HtlcID] = make(record.CustomSet) + } + + value := row.Value + if value == nil { + value = []byte{} + } + + customRecords[row.HtlcID][uint64(row.Key)] = value + } + + // Now fetch all the AMP HTLCs for this invoice or the one matching the + // provided set ID. + ampHtlcRows, err := db.FetchAMPSubInvoiceHTLCs( + ctx, sqlc.FetchAMPSubInvoiceHTLCsParams{ + InvoiceID: invoiceID, + SetID: paramSetID, + }, + ) + if err != nil { + return nil, nil, err + } + + ampHtlcs := make(map[models.CircuitKey]*invpkg.InvoiceHTLC) + for _, row := range ampHtlcRows { + uint64ChanID, err := strconv.ParseUint(row.ChanID, 10, 64) + if err != nil { + return nil, nil, err + } + + chanID := lnwire.NewShortChanIDFromInt(uint64ChanID) + + if row.HtlcID < 0 { + return nil, nil, fmt.Errorf("invalid HTLC ID "+ + "value: %v", row.HtlcID) + } + + htlcID := uint64(row.HtlcID) + + circuitKey := invpkg.CircuitKey{ + ChanID: chanID, + HtlcID: htlcID, + } + + htlc := &invpkg.InvoiceHTLC{ + Amt: lnwire.MilliSatoshi(row.AmountMsat), + AcceptHeight: uint32(row.AcceptHeight), + AcceptTime: row.AcceptTime.Local(), + Expiry: uint32(row.ExpiryHeight), + State: invpkg.HtlcState(row.State), + } + + if row.TotalMppMsat.Valid { + htlc.MppTotalAmt = lnwire.MilliSatoshi( + row.TotalMppMsat.Int64, + ) + } + + if row.ResolveTime.Valid { + htlc.ResolveTime = row.ResolveTime.Time.Local() + } + + var ( + rootShare [32]byte + setID [32]byte + ) + + if len(row.RootShare) != 32 { + return nil, nil, fmt.Errorf("invalid root share "+ + "length: %d", len(row.RootShare)) + } + copy(rootShare[:], row.RootShare) + + if len(row.SetID) != 32 { + return nil, nil, fmt.Errorf("invalid set ID length: %d", + len(row.SetID)) + } + copy(setID[:], row.SetID) + + if row.ChildIndex < 0 || row.ChildIndex > math.MaxUint32 { + return nil, nil, fmt.Errorf("invalid child index "+ + "value: %v", row.ChildIndex) + } + + ampRecord := record.NewAMP( + rootShare, setID, uint32(row.ChildIndex), + ) + + htlc.AMP = &invpkg.InvoiceHtlcAMPData{ + Record: *ampRecord, + } + + if len(row.Hash) != 32 { + return nil, nil, fmt.Errorf("invalid hash length: %d", + len(row.Hash)) + } + copy(htlc.AMP.Hash[:], row.Hash) + + if row.Preimage != nil { + preimage, err := lntypes.MakePreimage(row.Preimage) + if err != nil { + return nil, nil, err + } + + htlc.AMP.Preimage = &preimage + } + + if _, ok := customRecords[row.ID]; ok { + htlc.CustomRecords = customRecords[row.ID] + } else { + htlc.CustomRecords = make(record.CustomSet) + } + + ampHtlcs[circuitKey] = htlc + } + + if len(ampHtlcs) > 0 { + for setID := range ampState { + var amtPaid lnwire.MilliSatoshi + invoiceKeys := make( + map[models.CircuitKey]struct{}, + ) + + for key, htlc := range ampHtlcs { + if htlc.AMP.Record.SetID() != setID { + continue + } + + invoiceKeys[key] = struct{}{} + + if htlc.State != invpkg.HtlcStateCanceled { //nolint: lll + amtPaid += htlc.Amt + } + } + + setState := ampState[setID] + setState.InvoiceKeys = invoiceKeys + setState.AmtPaid = amtPaid + ampState[setID] = setState + } + } + + return ampState, ampHtlcs, nil +} + +// LookupInvoice attempts to look up an invoice corresponding the passed in +// reference. The reference may be a payment hash, a payment address, or a set +// ID for an AMP sub invoice. If the invoice is found, we'll return the complete +// invoice. If the invoice is not found, then we'll return an ErrInvoiceNotFound +// error. +func (i *InvoiceStore) LookupInvoice(ctx context.Context, + ref invpkg.InvoiceRef) (invpkg.Invoice, error) { + + var ( + invoice *invpkg.Invoice + err error + ) + + readTxOpt := NewInvoiceQueryReadTx() + txErr := i.db.ExecTx(ctx, &readTxOpt, func(db InvoiceQueries) error { + invoice, err = i.fetchInvoice(ctx, db, ref) + + return err + }) + if txErr != nil { + return invpkg.Invoice{}, txErr + } + + return *invoice, nil +} + +// FetchPendingInvoices returns all the invoices that are currently in a +// "pending" state. An invoice is pending if it has been created but not yet +// settled or canceled. +func (i *InvoiceStore) FetchPendingInvoices(ctx context.Context) ( + map[lntypes.Hash]invpkg.Invoice, error) { + + invoices := make(map[lntypes.Hash]invpkg.Invoice) + + readTxOpt := NewInvoiceQueryReadTx() + err := i.db.ExecTx(ctx, &readTxOpt, func(db InvoiceQueries) error { + limit := queryPaginationLimit + + return queryWithLimit(func(offset int) (int, error) { + params := sqlc.FilterInvoicesParams{ + PendingOnly: true, + NumOffset: int32(offset), + NumLimit: int32(limit), + } + + rows, err := db.FilterInvoices(ctx, params) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("unable to get invoices "+ + "from db: %w", err) + } + + // Load all the information for the invoices. + for _, row := range rows { + hash, invoice, err := fetchInvoiceData( + ctx, db, row, nil, true, + ) + if err != nil { + return 0, err + } + + invoices[*hash] = *invoice + } + + return len(rows), nil + }, limit) + }) + if err != nil { + return nil, fmt.Errorf("unable to fetch pending invoices: %w", + err) + } + + return invoices, nil +} + +// InvoicesSettledSince can be used by callers to catch up any settled invoices +// they missed within the settled invoice time series. We'll return all known +// settled invoice that have a settle index higher than the passed +// sinceSettleIndex. +// +// NOTE: The index starts from 1. As a result we enforce that specifying a value +// below the starting index value is a noop. +func (i *InvoiceStore) InvoicesSettledSince(ctx context.Context, idx uint64) ( + []invpkg.Invoice, error) { + + var invoices []invpkg.Invoice + + if idx == 0 { + return invoices, nil + } + + readTxOpt := NewInvoiceQueryReadTx() + err := i.db.ExecTx(ctx, &readTxOpt, func(db InvoiceQueries) error { + settleIdx := idx + limit := queryPaginationLimit + + err := queryWithLimit(func(offset int) (int, error) { + params := sqlc.FilterInvoicesParams{ + SettleIndexGet: sqlInt64(settleIdx + 1), + NumLimit: int32(limit), + NumOffset: int32(offset), + } + + rows, err := db.FilterInvoices(ctx, params) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("unable to get invoices "+ + "from db: %w", err) + } + + // Load all the information for the invoices. + for _, row := range rows { + _, invoice, err := fetchInvoiceData( + ctx, db, row, nil, true, + ) + if err != nil { + return 0, fmt.Errorf("unable to fetch "+ + "invoice(id=%d) from db: %w", + row.ID, err) + } + + invoices = append(invoices, *invoice) + } + + settleIdx += uint64(limit) + + return len(rows), nil + }, limit) + if err != nil { + return err + } + + // Now fetch all the AMP sub invoices that were settled since + // the provided index. + ampInvoices, err := i.db.FetchSettledAMPSubInvoices( + ctx, sqlc.FetchSettledAMPSubInvoicesParams{ + SettleIndexGet: sqlInt64(idx + 1), + }, + ) + if err != nil { + return err + } + + for _, ampInvoice := range ampInvoices { + // Convert the row to a sqlc.Invoice so we can use the + // existing fetchInvoiceData function. + sqlInvoice := sqlc.Invoice{ + ID: ampInvoice.ID, + Hash: ampInvoice.Hash, + Preimage: ampInvoice.Preimage, + SettleIndex: ampInvoice.AmpSettleIndex, + SettledAt: ampInvoice.AmpSettledAt, + Memo: ampInvoice.Memo, + AmountMsat: ampInvoice.AmountMsat, + CltvDelta: ampInvoice.CltvDelta, + Expiry: ampInvoice.Expiry, + PaymentAddr: ampInvoice.PaymentAddr, + PaymentRequest: ampInvoice.PaymentRequest, + State: ampInvoice.State, + AmountPaidMsat: ampInvoice.AmountPaidMsat, + IsAmp: ampInvoice.IsAmp, + IsHodl: ampInvoice.IsHodl, + IsKeysend: ampInvoice.IsKeysend, + CreatedAt: ampInvoice.CreatedAt.UTC(), + } + + // Fetch the state and HTLCs for this AMP sub invoice. + _, invoice, err := fetchInvoiceData( + ctx, db, sqlInvoice, + (*[32]byte)(ampInvoice.SetID), true, + ) + if err != nil { + return fmt.Errorf("unable to fetch "+ + "AMP invoice(id=%d) from db: %w", + ampInvoice.ID, err) + } + + invoices = append(invoices, *invoice) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("unable to get invoices settled since "+ + "index (excluding) %d: %w", idx, err) + } + + return invoices, nil +} + +// InvoicesAddedSince can be used by callers to seek into the event time series +// of all the invoices added in the database. This method will return all +// invoices with an add index greater than the specified sinceAddIndex. +// +// NOTE: The index starts from 1. As a result we enforce that specifying a value +// below the starting index value is a noop. +func (i *InvoiceStore) InvoicesAddedSince(ctx context.Context, idx uint64) ( + []invpkg.Invoice, error) { + + var result []invpkg.Invoice + + if idx == 0 { + return result, nil + } + + readTxOpt := NewInvoiceQueryReadTx() + err := i.db.ExecTx(ctx, &readTxOpt, func(db InvoiceQueries) error { + addIdx := idx + limit := queryPaginationLimit + + return queryWithLimit(func(offset int) (int, error) { + params := sqlc.FilterInvoicesParams{ + AddIndexGet: sqlInt64(addIdx + 1), + NumLimit: int32(limit), + NumOffset: int32(offset), + } + + rows, err := db.FilterInvoices(ctx, params) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("unable to get invoices "+ + "from db: %w", err) + } + + // Load all the information for the invoices. + for _, row := range rows { + _, invoice, err := fetchInvoiceData( + ctx, db, row, nil, true, + ) + if err != nil { + return 0, err + } + + result = append(result, *invoice) + } + + addIdx += uint64(limit) + + return len(rows), nil + }, limit) + }) + + if err != nil { + return nil, fmt.Errorf("unable to get invoices added since "+ + "index %d: %w", idx, err) + } + + return result, nil +} + +// QueryInvoices allows a caller to query the invoice database for invoices +// within the specified add index range. +func (i *InvoiceStore) QueryInvoices(ctx context.Context, + q invpkg.InvoiceQuery) (invpkg.InvoiceSlice, error) { + + var invoices []invpkg.Invoice + + if q.NumMaxInvoices == 0 { + return invpkg.InvoiceSlice{}, fmt.Errorf("max invoices must " + + "be non-zero") + } + + readTxOpt := NewInvoiceQueryReadTx() + err := i.db.ExecTx(ctx, &readTxOpt, func(db InvoiceQueries) error { + limit := queryPaginationLimit + + return queryWithLimit(func(offset int) (int, error) { + params := sqlc.FilterInvoicesParams{ + NumOffset: int32(offset), + NumLimit: int32(limit), + PendingOnly: q.PendingOnly, + } + + if !q.Reversed { + // The invoice with index offset id must not be + // included in the results. + params.AddIndexGet = sqlInt64( + q.IndexOffset + uint64(offset) + 1, + ) + } + + if q.Reversed { + idx := int32(q.IndexOffset) + + // If the index offset was not set, we want to + // fetch from the lastest invoice. + if idx == 0 { + params.AddIndexLet = sqlInt64( + int64(math.MaxInt64), + ) + } else { + // The invoice with index offset id must + // not be included in the results. + params.AddIndexLet = sqlInt64( + idx - int32(offset) - 1, + ) + } + + params.Reverse = true + } + + if q.CreationDateStart != 0 { + params.CreatedAfter = sqlTime( + time.Unix(q.CreationDateStart, 0).UTC(), + ) + } + + if q.CreationDateEnd != 0 { + params.CreatedBefore = sqlTime( + time.Unix(q.CreationDateEnd, 0).UTC(), + ) + } + + rows, err := db.FilterInvoices(ctx, params) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("unable to get invoices "+ + "from db: %w", err) + } + + // Load all the information for the invoices. + for _, row := range rows { + _, invoice, err := fetchInvoiceData( + ctx, db, row, nil, true, + ) + if err != nil { + return 0, err + } + + invoices = append(invoices, *invoice) + + if len(invoices) == int(q.NumMaxInvoices) { + return 0, nil + } + } + + return len(rows), nil + }, limit) + }) + if err != nil { + return invpkg.InvoiceSlice{}, fmt.Errorf("unable to query "+ + "invoices: %w", err) + } + + if len(invoices) == 0 { + return invpkg.InvoiceSlice{ + InvoiceQuery: q, + }, nil + } + + // If we iterated through the add index in reverse order, then + // we'll need to reverse the slice of invoices to return them in + // forward order. + if q.Reversed { + numInvoices := len(invoices) + for i := 0; i < numInvoices/2; i++ { + reverse := numInvoices - i - 1 + invoices[i], invoices[reverse] = + invoices[reverse], invoices[i] + } + } + + res := invpkg.InvoiceSlice{ + InvoiceQuery: q, + Invoices: invoices, + FirstIndexOffset: invoices[0].AddIndex, + LastIndexOffset: invoices[len(invoices)-1].AddIndex, + } + + return res, nil +} + +// sqlInvoiceUpdater is the implementation of the InvoiceUpdater interface using +// a SQL database as the backend. +type sqlInvoiceUpdater struct { + db InvoiceQueries + ctx context.Context //nolint:containedctx + invoice *invpkg.Invoice + updateTime time.Time +} + +// AddHtlc adds a new htlc to the invoice. +func (s *sqlInvoiceUpdater) AddHtlc(circuitKey models.CircuitKey, + newHtlc *invpkg.InvoiceHTLC) error { + + htlcPrimaryKeyID, err := s.db.InsertInvoiceHTLC( + s.ctx, sqlc.InsertInvoiceHTLCParams{ + HtlcID: int64(circuitKey.HtlcID), + ChanID: strconv.FormatUint( + circuitKey.ChanID.ToUint64(), 10, + ), + AmountMsat: int64(newHtlc.Amt), + TotalMppMsat: sql.NullInt64{ + Int64: int64(newHtlc.MppTotalAmt), + Valid: newHtlc.MppTotalAmt != 0, + }, + AcceptHeight: int32(newHtlc.AcceptHeight), + AcceptTime: newHtlc.AcceptTime.UTC(), + ExpiryHeight: int32(newHtlc.Expiry), + State: int16(newHtlc.State), + InvoiceID: int64(s.invoice.AddIndex), + }, + ) + if err != nil { + return err + } + + for key, value := range newHtlc.CustomRecords { + err = s.db.InsertInvoiceHTLCCustomRecord( + s.ctx, sqlc.InsertInvoiceHTLCCustomRecordParams{ + // TODO(bhandras): schema might be wrong here + // as the custom record key is an uint64. + Key: int64(key), + Value: value, + HtlcID: htlcPrimaryKeyID, + }, + ) + if err != nil { + return err + } + } + + if newHtlc.AMP != nil { + setID := newHtlc.AMP.Record.SetID() + + upsertResult, err := s.db.UpsertAMPSubInvoice( + s.ctx, sqlc.UpsertAMPSubInvoiceParams{ + SetID: setID[:], + CreatedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + }, + ) + if err != nil { + mappedSQLErr := MapSQLError(err) + var uniqueConstraintErr *ErrSQLUniqueConstraintViolation + if errors.As(mappedSQLErr, &uniqueConstraintErr) { + return invpkg.ErrDuplicateSetID{ + SetID: setID, + } + } + + return err + } + + // If we're just inserting the AMP invoice, we'll get a non + // zero rows affected count. + rowsAffected, err := upsertResult.RowsAffected() + if err != nil { + return err + } + if rowsAffected != 0 { + // If we're inserting a new AMP invoice, we'll also + // insert a new invoice event. + err = s.db.OnAMPSubInvoiceCreated( + s.ctx, sqlc.OnAMPSubInvoiceCreatedParams{ + AddedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + SetID: setID[:], + }, + ) + if err != nil { + return err + } + } + + rootShare := newHtlc.AMP.Record.RootShare() + + ampHtlcParams := sqlc.InsertAMPSubInvoiceHTLCParams{ + InvoiceID: int64(s.invoice.AddIndex), + SetID: setID[:], + HtlcID: htlcPrimaryKeyID, + RootShare: rootShare[:], + ChildIndex: int64( + newHtlc.AMP.Record.ChildIndex(), + ), + Hash: newHtlc.AMP.Hash[:], + } + + if newHtlc.AMP.Preimage != nil { + ampHtlcParams.Preimage = newHtlc.AMP.Preimage[:] + } + + err = s.db.InsertAMPSubInvoiceHTLC(s.ctx, ampHtlcParams) + if err != nil { + return err + } + } + + return nil +} + +// ResolveHtlc marks an htlc as resolved with the given state. +func (s *sqlInvoiceUpdater) ResolveHtlc(circuitKey models.CircuitKey, + state invpkg.HtlcState, resolveTime time.Time) error { + + return s.db.UpdateInvoiceHTLC(s.ctx, sqlc.UpdateInvoiceHTLCParams{ + HtlcID: int64(circuitKey.HtlcID), + ChanID: strconv.FormatUint( + circuitKey.ChanID.ToUint64(), 10, + ), + InvoiceID: int64(s.invoice.AddIndex), + State: int16(state), + ResolveTime: sqlTime(resolveTime.UTC()), + }) +} + +// AddAmpHtlcPreimage adds a preimage of an AMP htlc to the AMP sub invoice +// identified by the setID. +func (s *sqlInvoiceUpdater) AddAmpHtlcPreimage(setID [32]byte, + circuitKey models.CircuitKey, preimage lntypes.Preimage) error { + + result, err := s.db.UpdateAMPSubInvoiceHTLCPreimage( + s.ctx, sqlc.UpdateAMPSubInvoiceHTLCPreimageParams{ + InvoiceID: int64(s.invoice.AddIndex), + SetID: setID[:], + HtlcID: int64(circuitKey.HtlcID), + Preimage: preimage[:], + }, + ) + if err != nil { + return err + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return err + } + if rowsAffected == 0 { + return invpkg.ErrInvoiceNotFound + } + + return nil +} + +// UpdateInvoiceState updates the invoice state to the new state. +func (s *sqlInvoiceUpdater) UpdateInvoiceState( + newState invpkg.ContractState, preimage *lntypes.Preimage) error { + + var ( + settleIndex sql.NullInt64 + settledAt sql.NullTime + ) + + switch newState { + case invpkg.ContractSettled: + nextSettleIndex, err := s.db.NextInvoiceSettleIndex(s.ctx) + if err != nil { + return err + } + + settleIndex = sqlInt64(nextSettleIndex) + + // If the invoice is settled, we'll also update the settle time. + settledAt = sqlTime(s.updateTime.UTC()) + + err = s.db.OnInvoiceSettled( + s.ctx, sqlc.OnInvoiceSettledParams{ + AddedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + }, + ) + if err != nil { + return err + } + + case invpkg.ContractCanceled: + err := s.db.OnInvoiceCanceled( + s.ctx, sqlc.OnInvoiceCanceledParams{ + AddedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + }, + ) + if err != nil { + return err + } + } + + params := sqlc.UpdateInvoiceStateParams{ + ID: int64(s.invoice.AddIndex), + State: int16(newState), + SettleIndex: settleIndex, + SettledAt: settledAt, + } + + if preimage != nil { + params.Preimage = preimage[:] + } + + result, err := s.db.UpdateInvoiceState(s.ctx, params) + if err != nil { + return err + } + rowsAffected, err := result.RowsAffected() + if err != nil { + return err + } + + if rowsAffected == 0 { + return invpkg.ErrInvoiceNotFound + } + + if settleIndex.Valid { + s.invoice.SettleIndex = uint64(settleIndex.Int64) + s.invoice.SettleDate = s.updateTime + } + + return nil +} + +// UpdateInvoiceAmtPaid updates the invoice amount paid to the new amount. +func (s *sqlInvoiceUpdater) UpdateInvoiceAmtPaid( + amtPaid lnwire.MilliSatoshi) error { + + _, err := s.db.UpdateInvoiceAmountPaid( + s.ctx, sqlc.UpdateInvoiceAmountPaidParams{ + ID: int64(s.invoice.AddIndex), + AmountPaidMsat: int64(amtPaid), + }, + ) + + return err +} + +// UpdateAmpState updates the state of the AMP sub invoice identified by the +// setID. +func (s *sqlInvoiceUpdater) UpdateAmpState(setID [32]byte, + newState invpkg.InvoiceStateAMP, _ models.CircuitKey) error { + + var ( + settleIndex sql.NullInt64 + settledAt sql.NullTime + ) + + switch newState.State { + case invpkg.HtlcStateSettled: + nextSettleIndex, err := s.db.NextInvoiceSettleIndex(s.ctx) + if err != nil { + return err + } + + settleIndex = sqlInt64(nextSettleIndex) + + // If the invoice is settled, we'll also update the settle time. + settledAt = sqlTime(s.updateTime.UTC()) + + err = s.db.OnAMPSubInvoiceSettled( + s.ctx, sqlc.OnAMPSubInvoiceSettledParams{ + AddedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + SetID: setID[:], + }, + ) + if err != nil { + return err + } + + case invpkg.HtlcStateCanceled: + err := s.db.OnAMPSubInvoiceCanceled( + s.ctx, sqlc.OnAMPSubInvoiceCanceledParams{ + AddedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + SetID: setID[:], + }, + ) + if err != nil { + return err + } + } + + err := s.db.UpdateAMPSubInvoiceState( + s.ctx, sqlc.UpdateAMPSubInvoiceStateParams{ + SetID: setID[:], + State: int16(newState.State), + SettleIndex: settleIndex, + SettledAt: settledAt, + }, + ) + if err != nil { + return err + } + + return nil +} + +// Finalize finalizes the update before it is written to the database. Note that +// we don't use this directly in the SQL implementation, so the function is just +// a stub. +func (s *sqlInvoiceUpdater) Finalize(_ invpkg.UpdateType) error { + return nil +} + +// UpdateInvoice attempts to update an invoice corresponding to the passed +// reference. If an invoice matching the passed reference doesn't exist within +// the database, then the action will fail with ErrInvoiceNotFound error. +// +// The update is performed inside the same database transaction that fetches the +// invoice and is therefore atomic. The fields to update are controlled by the +// supplied callback. +func (i *InvoiceStore) UpdateInvoice(ctx context.Context, ref invpkg.InvoiceRef, + _ *invpkg.SetID, callback invpkg.InvoiceUpdateCallback) ( + *invpkg.Invoice, error) { + + var updatedInvoice *invpkg.Invoice + + txOpt := InvoiceQueriesTxOptions{readOnly: false} + txErr := i.db.ExecTx(ctx, &txOpt, func(db InvoiceQueries) error { + invoice, err := i.fetchInvoice(ctx, db, ref) + if err != nil { + return err + } + + updateTime := i.clock.Now() + updater := &sqlInvoiceUpdater{ + db: db, + ctx: ctx, + invoice: invoice, + updateTime: updateTime, + } + + payHash := ref.PayHash() + updatedInvoice, err = invpkg.UpdateInvoice( + payHash, invoice, updateTime, callback, updater, + ) + + return err + }) + if txErr != nil { + // If the invoice is already settled, we'll return the + // (unchanged) invoice and the ErrInvoiceAlreadySettled error. + if errors.Is(txErr, invpkg.ErrInvoiceAlreadySettled) { + return updatedInvoice, txErr + } + + return nil, txErr + } + + return updatedInvoice, nil +} + +// DeleteInvoice attempts to delete the passed invoices and all their related +// data from the database in one transaction. +func (i *InvoiceStore) DeleteInvoice(ctx context.Context, + invoicesToDelete []invpkg.InvoiceDeleteRef) error { + + // All the InvoiceDeleteRef instances include the add index of the + // invoice. The rest was added to ensure that the invoices were deleted + // properly in the kv database. When we have fully migrated we can + // remove the rest of the fields. + for _, ref := range invoicesToDelete { + if ref.AddIndex == 0 { + return fmt.Errorf("unable to delete invoice using a "+ + "ref without AddIndex set: %v", ref) + } + } + + var writeTxOpt InvoiceQueriesTxOptions + err := i.db.ExecTx(ctx, &writeTxOpt, func(db InvoiceQueries) error { + for _, ref := range invoicesToDelete { + params := sqlc.DeleteInvoiceParams{ + AddIndex: sqlInt64(ref.AddIndex), + } + + if ref.SettleIndex != 0 { + params.SettleIndex = sqlInt64(ref.SettleIndex) + } + + if ref.PayHash != lntypes.ZeroHash { + params.Hash = ref.PayHash[:] + } + + result, err := db.DeleteInvoice(ctx, params) + if err != nil { + return fmt.Errorf("unable to delete "+ + "invoice(%v): %w", ref.AddIndex, err) + } + rowsAffected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("unable to get rows "+ + "affected: %w", err) + } + if rowsAffected == 0 { + return fmt.Errorf("%w: %v", + invpkg.ErrInvoiceNotFound, ref.AddIndex) + } + } + + return nil + }) + + if err != nil { + return fmt.Errorf("unable to delete invoices: %w", err) + } + + return nil +} + +// DeleteCanceledInvoices removes all canceled invoices from the database. +func (i *InvoiceStore) DeleteCanceledInvoices(ctx context.Context) error { + var writeTxOpt InvoiceQueriesTxOptions + err := i.db.ExecTx(ctx, &writeTxOpt, func(db InvoiceQueries) error { + _, err := db.DeleteCanceledInvoices(ctx) + if err != nil { + return fmt.Errorf("unable to delete canceled "+ + "invoices: %w", err) + } + + return nil + }) + if err != nil { + return fmt.Errorf("unable to delete invoices: %w", err) + } + + return nil +} + +// fetchInvoiceData fetches additional data for the given invoice. If the +// invoice is AMP and the setID is not nil, then it will also fetch the AMP +// state and HTLCs for the given setID, otherwise for all AMP sub invoices of +// the invoice. If fetchAmpHtlcs is true, it will also fetch the AMP HTLCs. +func fetchInvoiceData(ctx context.Context, db InvoiceQueries, + row sqlc.Invoice, setID *[32]byte, fetchAmpHtlcs bool) (*lntypes.Hash, + *invpkg.Invoice, error) { + + // Unmarshal the common data. + hash, invoice, err := unmarshalInvoice(row) + if err != nil { + return nil, nil, fmt.Errorf("unable to unmarshal "+ + "invoice(id=%d) from db: %w", row.ID, err) + } + + // Fetch the invoice features. + features, err := getInvoiceFeatures(ctx, db, row.ID) + if err != nil { + return nil, nil, err + } + + invoice.Terms.Features = features + + // If this is an AMP invoice, we'll need fetch the AMP state along + // with the HTLCs (if requested). + if invoice.IsAMP() { + invoiceID := int64(invoice.AddIndex) + ampState, ampHtlcs, err := fetchAmpState( + ctx, db, invoiceID, setID, fetchAmpHtlcs, + ) + if err != nil { + return nil, nil, err + } + + invoice.AMPState = ampState + invoice.Htlcs = ampHtlcs + + return hash, invoice, nil + } + + // Otherwise simply fetch the invoice HTLCs. + htlcs, err := getInvoiceHtlcs(ctx, db, row.ID) + if err != nil { + return nil, nil, err + } + + if len(htlcs) > 0 { + invoice.Htlcs = htlcs + var amountPaid lnwire.MilliSatoshi + for _, htlc := range htlcs { + if htlc.State == invpkg.HtlcStateSettled { + amountPaid += htlc.Amt + } + } + invoice.AmtPaid = amountPaid + } + + return hash, invoice, nil +} + +// getInvoiceFeatures fetches the invoice features for the given invoice id. +func getInvoiceFeatures(ctx context.Context, db InvoiceQueries, + invoiceID int64) (*lnwire.FeatureVector, error) { + + rows, err := db.GetInvoiceFeatures(ctx, invoiceID) + if err != nil { + return nil, fmt.Errorf("unable to get invoice features: %w", + err) + } + + features := lnwire.EmptyFeatureVector() + for _, feature := range rows { + features.Set(lnwire.FeatureBit(feature.Feature)) + } + + return features, nil +} + +// getInvoiceHtlcs fetches the invoice htlcs for the given invoice id. +func getInvoiceHtlcs(ctx context.Context, db InvoiceQueries, + invoiceID int64) (map[invpkg.CircuitKey]*invpkg.InvoiceHTLC, error) { + + htlcRows, err := db.GetInvoiceHTLCs(ctx, invoiceID) + if err != nil { + return nil, fmt.Errorf("unable to get invoice htlcs: %w", err) + } + + // We have no htlcs to unmarshal. + if len(htlcRows) == 0 { + return nil, nil + } + + crRows, err := db.GetInvoiceHTLCCustomRecords(ctx, invoiceID) + if err != nil { + return nil, fmt.Errorf("unable to get custom records for "+ + "invoice htlcs: %w", err) + } + + cr := make(map[int64]record.CustomSet, len(crRows)) + for _, row := range crRows { + if _, ok := cr[row.HtlcID]; !ok { + cr[row.HtlcID] = make(record.CustomSet) + } + + value := row.Value + if value == nil { + value = []byte{} + } + cr[row.HtlcID][uint64(row.Key)] = value + } + + htlcs := make(map[invpkg.CircuitKey]*invpkg.InvoiceHTLC, len(htlcRows)) + + for _, row := range htlcRows { + circuiteKey, htlc, err := unmarshalInvoiceHTLC(row) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal "+ + "htlc(%d): %w", row.ID, err) + } + + if customRecords, ok := cr[row.ID]; ok { + htlc.CustomRecords = customRecords + } else { + htlc.CustomRecords = make(record.CustomSet) + } + + htlcs[circuiteKey] = htlc + } + + return htlcs, nil +} + +// unmarshalInvoice converts an InvoiceRow to an Invoice. +func unmarshalInvoice(row sqlc.Invoice) (*lntypes.Hash, *invpkg.Invoice, + error) { + + var ( + settleIndex int64 + settledAt time.Time + memo []byte + paymentRequest []byte + preimage *lntypes.Preimage + paymentAddr [32]byte + ) + + hash, err := lntypes.MakeHash(row.Hash) + if err != nil { + return nil, nil, err + } + + if row.SettleIndex.Valid { + settleIndex = row.SettleIndex.Int64 + } + + if row.SettledAt.Valid { + settledAt = row.SettledAt.Time.Local() + } + + if row.Memo.Valid { + memo = []byte(row.Memo.String) + } + + // Keysend payments will have this field empty. + if row.PaymentRequest.Valid { + paymentRequest = []byte(row.PaymentRequest.String) + } else { + paymentRequest = []byte{} + } + + // We may not have the preimage if this a hodl invoice. + if row.Preimage != nil { + preimage = &lntypes.Preimage{} + copy(preimage[:], row.Preimage) + } + + copy(paymentAddr[:], row.PaymentAddr) + + var cltvDelta int32 + if row.CltvDelta.Valid { + cltvDelta = row.CltvDelta.Int32 + } + + invoice := &invpkg.Invoice{ + SettleIndex: uint64(settleIndex), + SettleDate: settledAt, + Memo: memo, + PaymentRequest: paymentRequest, + CreationDate: row.CreatedAt.Local(), + Terms: invpkg.ContractTerm{ + FinalCltvDelta: cltvDelta, + Expiry: time.Duration(row.Expiry), + PaymentPreimage: preimage, + Value: lnwire.MilliSatoshi(row.AmountMsat), + PaymentAddr: paymentAddr, + }, + AddIndex: uint64(row.ID), + State: invpkg.ContractState(row.State), + AmtPaid: lnwire.MilliSatoshi(row.AmountPaidMsat), + Htlcs: make(map[models.CircuitKey]*invpkg.InvoiceHTLC), + AMPState: invpkg.AMPInvoiceState{}, + HodlInvoice: row.IsHodl, + } + + return &hash, invoice, nil +} + +// unmarshalInvoiceHTLC converts an sqlc.InvoiceHtlc to an InvoiceHTLC. +func unmarshalInvoiceHTLC(row sqlc.InvoiceHtlc) (invpkg.CircuitKey, + *invpkg.InvoiceHTLC, error) { + + uint64ChanID, err := strconv.ParseUint(row.ChanID, 10, 64) + if err != nil { + return invpkg.CircuitKey{}, nil, err + } + + chanID := lnwire.NewShortChanIDFromInt(uint64ChanID) + + if row.HtlcID < 0 { + return invpkg.CircuitKey{}, nil, fmt.Errorf("invalid uint64 "+ + "value: %v", row.HtlcID) + } + + htlcID := uint64(row.HtlcID) + + circuitKey := invpkg.CircuitKey{ + ChanID: chanID, + HtlcID: htlcID, + } + + htlc := &invpkg.InvoiceHTLC{ + Amt: lnwire.MilliSatoshi(row.AmountMsat), + AcceptHeight: uint32(row.AcceptHeight), + AcceptTime: row.AcceptTime.Local(), + Expiry: uint32(row.ExpiryHeight), + State: invpkg.HtlcState(row.State), + } + + if row.TotalMppMsat.Valid { + htlc.MppTotalAmt = lnwire.MilliSatoshi(row.TotalMppMsat.Int64) + } + + if row.ResolveTime.Valid { + htlc.ResolveTime = row.ResolveTime.Time.Local() + } + + return circuitKey, htlc, nil +} + +// queryWithLimit is a helper method that can be used to query the database +// using a limit and offset. The passed query function should return the number +// of rows returned and an error if any. +func queryWithLimit(query func(int) (int, error), limit int) error { + offset := 0 + for { + rows, err := query(offset) + if err != nil { + return err + } + + if rows < limit { + return nil + } + + offset += limit + } +} From 39211d25ede3e53152eab8b9ff1114264717eb95 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 22 Nov 2023 17:08:41 +0100 Subject: [PATCH 07/21] invoices: make test HTLC circuit keys unique --- invoices/invoices_test.go | 67 +++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/invoices/invoices_test.go b/invoices/invoices_test.go index caf58f13a..4d5ed96bf 100644 --- a/invoices/invoices_test.go +++ b/invoices/invoices_test.go @@ -77,11 +77,13 @@ func randInvoice(value lnwire.MilliSatoshi) (*invpkg.Invoice, error) { } // settleTestInvoice settles a test invoice. -func settleTestInvoice(invoice *invpkg.Invoice, settleIndex uint64) { +func settleTestInvoice(invoice *invpkg.Invoice, htlcID uint64, + settleIndex uint64) { + invoice.SettleDate = testNow invoice.AmtPaid = invoice.Terms.Value invoice.State = invpkg.ContractSettled - invoice.Htlcs[models.CircuitKey{}] = &invpkg.InvoiceHTLC{ + invoice.Htlcs[models.CircuitKey{HtlcID: htlcID}] = &invpkg.InvoiceHTLC{ Amt: invoice.Terms.Value, AcceptTime: testNow, ResolveTime: testNow, @@ -333,7 +335,7 @@ func testInvoiceWorkflowImpl(t *testing.T, test invWorkflowTest, // now have the settled bit toggle to true and a non-default // SettledDate payAmt := fakeInvoice.Terms.Value * 2 - _, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(payAmt)) + _, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(0, payAmt)) require.NoError(t, err, "unable to settle invoice") dbInvoice2, err := db.LookupInvoice(ctxb, ref) require.NoError(t, err, "unable to fetch invoice") @@ -902,8 +904,11 @@ func testInvoiceAddTimeSeries(t *testing.T, _, err = db.InvoicesSettledSince(ctxb, 0) require.NoError(t, err) - var settledInvoices []invpkg.Invoice - var settleIndex uint64 = 1 + var ( + settledInvoices []invpkg.Invoice + settleIndex uint64 = 1 + htlcID uint64 = 0 + ) // We'll now only settle the latter half of each of those invoices. for i := 10; i < len(invoices); i++ { invoice := &invoices[i] @@ -912,15 +917,18 @@ func testInvoiceAddTimeSeries(t *testing.T, ref := invpkg.InvoiceRefByHash(paymentHash) _, err := db.UpdateInvoice( - ctxb, ref, nil, getUpdateInvoice(invoice.Terms.Value), + ctxb, ref, nil, getUpdateInvoice( + htlcID, invoice.Terms.Value, + ), ) if err != nil { t.Fatalf("unable to settle invoice: %v", err) } // Create the settled invoice for the expectation set. - settleTestInvoice(invoice, settleIndex) + settleTestInvoice(invoice, htlcID, settleIndex) settleIndex++ + htlcID++ settledInvoices = append(settledInvoices, *invoice) } @@ -1158,7 +1166,11 @@ func testFetchPendingInvoices(t *testing.T, require.Empty(t, pending) const numInvoices = 20 - var settleIndex uint64 = 1 + var ( + settleIndex uint64 = 1 + htlcID uint64 = 0 + ) + pendingInvoices := make(map[lntypes.Hash]invpkg.Invoice) for i := 1; i <= numInvoices; i++ { @@ -1182,11 +1194,14 @@ func testFetchPendingInvoices(t *testing.T, } ref := invpkg.InvoiceRefByHash(paymentHash) - _, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(amt)) + _, err = db.UpdateInvoice( + ctxb, ref, nil, getUpdateInvoice(htlcID, amt), + ) require.NoError(t, err) - settleTestInvoice(invoice, settleIndex) + settleTestInvoice(invoice, htlcID, settleIndex) settleIndex++ + htlcID++ } // Fetch all pending invoices. @@ -1219,7 +1234,7 @@ func testDuplicateSettleInvoice(t *testing.T, // With the invoice in the DB, we'll now attempt to settle the invoice. ref := invpkg.InvoiceRefByHash(payHash) dbInvoice, err := db.UpdateInvoice( - ctxb, ref, nil, getUpdateInvoice(amt), + ctxb, ref, nil, getUpdateInvoice(0, amt), ) require.NoError(t, err, "unable to settle invoice") @@ -1244,7 +1259,9 @@ func testDuplicateSettleInvoice(t *testing.T, // If we try to settle the invoice again, then we should get the very // same invoice back, but with an error this time. - dbInvoice, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(amt)) + dbInvoice, err = db.UpdateInvoice( + ctxb, ref, nil, getUpdateInvoice(1, amt), + ) require.ErrorIs(t, err, invpkg.ErrInvoiceAlreadySettled) if dbInvoice == nil { @@ -1269,9 +1286,12 @@ func testQueryInvoices(t *testing.T, // assume that the index of the invoice within the database is the same // as the amount of the invoice itself. const numInvoices = 50 - var settleIndex uint64 = 1 - var invoices []invpkg.Invoice - var pendingInvoices []invpkg.Invoice + var ( + settleIndex uint64 = 1 + htlcID uint64 = 0 + invoices []invpkg.Invoice + pendingInvoices []invpkg.Invoice + ) ctxb := context.Background() for i := 1; i <= numInvoices; i++ { @@ -1296,15 +1316,16 @@ func testQueryInvoices(t *testing.T, if i%2 == 0 { ref := invpkg.InvoiceRefByHash(paymentHash) _, err := db.UpdateInvoice( - ctxb, ref, nil, getUpdateInvoice(amt), + ctxb, ref, nil, getUpdateInvoice(htlcID, amt), ) if err != nil { t.Fatalf("unable to settle invoice: %v", err) } // Create the settled invoice for the expectation set. - settleTestInvoice(invoice, settleIndex) + settleTestInvoice(invoice, htlcID, settleIndex) settleIndex++ + htlcID++ } else { pendingInvoices = append(pendingInvoices, *invoice) } @@ -1662,7 +1683,9 @@ func testQueryInvoices(t *testing.T, // getUpdateInvoice returns an invoice update callback that, when called, // settles the invoice with the given amount. -func getUpdateInvoice(amt lnwire.MilliSatoshi) invpkg.InvoiceUpdateCallback { +func getUpdateInvoice(htlcID uint64, + amt lnwire.MilliSatoshi) invpkg.InvoiceUpdateCallback { + return func(invoice *invpkg.Invoice) (*invpkg.InvoiceUpdateDesc, error) { @@ -1672,7 +1695,7 @@ func getUpdateInvoice(amt lnwire.MilliSatoshi) invpkg.InvoiceUpdateCallback { noRecords := make(record.CustomSet) htlcs := map[models.CircuitKey]*invpkg.HtlcAcceptDesc{ - {}: { + {HtlcID: htlcID}: { Amt: amt, CustomRecords: noRecords, }, @@ -2366,7 +2389,7 @@ func testUnexpectedInvoicePreimage(t *testing.T, // payment hash against which to validate the preimage. _, err = db.UpdateInvoice( ctxb, invpkg.InvoiceRefByAddr(invoice.Terms.PaymentAddr), nil, - getUpdateInvoice(invoice.Terms.Value), + getUpdateInvoice(0, invoice.Terms.Value), ) // Assert that we get ErrUnexpectedInvoicePreimage. @@ -2501,7 +2524,9 @@ func testDeleteInvoices(t *testing.T, if i == 1 { invoice, err = db.UpdateInvoice( ctxb, invpkg.InvoiceRefByHash(paymentHash), nil, - getUpdateInvoice(invoice.Terms.Value), + getUpdateInvoice( + uint64(i), invoice.Terms.Value, + ), ) require.NoError(t, err, "unable to settle invoice") } From 734d3a9d661988905cf0a0ba290d92dfc7d15ccc Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 22 Nov 2023 17:11:47 +0100 Subject: [PATCH 08/21] invoices: small test fixes --- invoices/invoices_test.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/invoices/invoices_test.go b/invoices/invoices_test.go index 4d5ed96bf..9e850fcbc 100644 --- a/invoices/invoices_test.go +++ b/invoices/invoices_test.go @@ -514,7 +514,7 @@ func testFailInvoiceLookupMPPPayAddrOnly(t *testing.T, // for the given HTLC. ref := invpkg.InvoiceRefByHashAndAddr(payHash, payAddr) _, err = db.LookupInvoice(ctxb, ref) - require.Equal(t, invpkg.ErrInvoiceNotFound, err) + require.ErrorIs(t, err, invpkg.ErrInvoiceNotFound) } // testInvRefEquivocation asserts that retrieving or updating an invoice using @@ -692,7 +692,7 @@ func testInvoiceCancelSingleHtlcAMP(t *testing.T, // At this point, we should detect that 3k satoshis total has been // paid. - require.Equal(t, dbInvoice.AmtPaid, amt*3) + require.EqualValues(t, amt*3, dbInvoice.AmtPaid) callback := func( invoice *invpkg.Invoice) (*invpkg.InvoiceUpdateDesc, error) { @@ -968,7 +968,10 @@ func testInvoiceAddTimeSeries(t *testing.T, t.Fatalf("unable to query: %v", err) } - require.Equal(t, len(query.resp), len(resp)) + require.Equal( + t, len(query.resp), len(resp), + fmt.Sprintf("test: #%v", i), + ) for j := 0; j < len(query.resp); j++ { require.Equal(t, @@ -1146,7 +1149,11 @@ func testSettleIndexAmpPayments(t *testing.T, AddIndex: testInvoice.AddIndex, }, }) - require.Nil(t, err) + require.NoError(t, err) + + settledInvoices, err = db.InvoicesSettledSince(ctxb, 0) + require.NoError(t, err) + require.Len(t, settledInvoices, 0) } // testFetchPendingInvoices tests that we can fetch all pending invoices from @@ -1670,7 +1677,8 @@ func testQueryInvoices(t *testing.T, t.Fatalf("unable to query invoice database: %v", err) } - require.Equal(t, len(testCase.expected), len(response.Invoices)) + require.Equal(t, len(testCase.expected), len(response.Invoices), + fmt.Sprintf("test: #%v", i)) for j, expected := range testCase.expected { require.Equal(t, From b10ce17cd4531ca57c881a65242a4b488496df56 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 22 Nov 2023 17:16:02 +0100 Subject: [PATCH 09/21] invoices: run InvoiceStore and InvoiceRegistry tests on SQLite too --- invoices/invoiceregistry_test.go | 22 +++++++++++++++++++++- invoices/invoices_test.go | 27 +++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/invoices/invoiceregistry_test.go b/invoices/invoiceregistry_test.go index 04fe310b5..a1204573c 100644 --- a/invoices/invoiceregistry_test.go +++ b/invoices/invoiceregistry_test.go @@ -3,6 +3,7 @@ package invoices_test import ( "context" "crypto/rand" + "database/sql" "fmt" "math" "testing" @@ -18,6 +19,7 @@ import ( "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/sqldb" "github.com/stretchr/testify/require" ) @@ -114,12 +116,30 @@ func TestInvoiceRegistry(t *testing.T) { return db, testClock } + makeSQLDB := func(t *testing.T) (invpkg.InvoiceDB, *clock.TestClock) { + db := sqldb.NewTestSqliteDB(t) + + executor := sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) sqldb.InvoiceQueries { + return db.BaseDB.WithTx(tx) + }, + ) + + testClock := clock.NewTestClock(testNow) + + return sqldb.NewInvoiceStore(executor, testClock), testClock + } + for _, test := range testList { test := test - t.Run(test.name, func(t *testing.T) { + t.Run(test.name+"_KV", func(t *testing.T) { test.test(t, makeKeyValueDB) }) + + t.Run(test.name+"_SQL", func(t *testing.T) { + test.test(t, makeSQLDB) + }) } } diff --git a/invoices/invoices_test.go b/invoices/invoices_test.go index 9e850fcbc..4296f0a54 100644 --- a/invoices/invoices_test.go +++ b/invoices/invoices_test.go @@ -3,6 +3,7 @@ package invoices_test import ( "context" "crypto/rand" + "database/sql" "fmt" "math" "testing" @@ -16,6 +17,7 @@ import ( "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/sqldb" "github.com/stretchr/testify/require" ) @@ -198,12 +200,30 @@ func TestInvoices(t *testing.T) { return db } + makeSQLDB := func(t *testing.T) invpkg.InvoiceDB { + db := sqldb.NewTestSqliteDB(t) + + executor := sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) sqldb.InvoiceQueries { + return db.BaseDB.WithTx(tx) + }, + ) + + return sqldb.NewInvoiceStore( + executor, clock.NewTestClock(testNow), + ) + } + for _, test := range testList { test := test - t.Run(test.name, func(t *testing.T) { + t.Run(test.name+"_KV", func(t *testing.T) { test.test(t, makeKeyValueDB) }) + + t.Run(test.name+"_SQL", func(t *testing.T) { + test.test(t, makeSQLDB) + }) } } @@ -2113,7 +2133,10 @@ func testSetIDIndex(t *testing.T, makeDB func(t *testing.T) invpkg.InvoiceDB) { ) _, err = db.UpdateInvoice( ctxb, ref2, (*invpkg.SetID)(setID), - updateAcceptAMPHtlc(0, amt, setID, true), + // Note: we need a unique HTLC ID here otherwise the update will + // be rejected as a duplicate (due to SQL unique constraint + // violation). + updateAcceptAMPHtlc(55, amt, setID, true), ) require.Equal(t, invpkg.ErrDuplicateSetID{SetID: *setID}, err) From 06730824a29ac703de58a83d61b3c885662edfa9 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Mon, 22 Jan 2024 17:51:21 +0100 Subject: [PATCH 10/21] sqldb: fixup PostgreSQL fixture to allow creating separate DBs per test This change will enable us to use a single PostgreSQL container instead of spawning new a one for each (parallel) unit test reducing overall test runtime. --- sqldb/postgres.go | 35 ++++++++++++++++++++++++++--------- sqldb/postgres_fixture.go | 1 + 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/sqldb/postgres.go b/sqldb/postgres.go index 9aac2a489..f5bf79d65 100644 --- a/sqldb/postgres.go +++ b/sqldb/postgres.go @@ -1,7 +1,10 @@ package sqldb import ( + "context" + "crypto/rand" "database/sql" + "encoding/hex" "fmt" "testing" "time" @@ -124,19 +127,33 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { } // NewTestPostgresDB is a helper function that creates a Postgres database for -// testing. -func NewTestPostgresDB(t *testing.T) *PostgresStore { +// testing using the given fixture. +func NewTestPostgresDB(t *testing.T, fixture *TestPgFixture) *PostgresStore { t.Helper() - t.Logf("Creating new Postgres DB for testing") + // Create random database name. + randBytes := make([]byte, 8) + _, err := rand.Read(randBytes) + if err != nil { + t.Fatal(err) + } - sqlFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime) - store, err := NewPostgresStore(sqlFixture.GetConfig()) + dbName := "test_" + hex.EncodeToString(randBytes) + + t.Logf("Creating new Postgres DB '%s' for testing", dbName) + + _, err = fixture.db.ExecContext( + context.Background(), "CREATE DATABASE "+dbName, + ) + if err != nil { + t.Fatal(err) + } + + cfg := fixture.GetConfig() + cfg.DBName = dbName + + store, err := NewPostgresStore(cfg) require.NoError(t, err) - t.Cleanup(func() { - sqlFixture.TearDown(t) - }) - return store } diff --git a/sqldb/postgres_fixture.go b/sqldb/postgres_fixture.go index 991aad980..313684f96 100644 --- a/sqldb/postgres_fixture.go +++ b/sqldb/postgres_fixture.go @@ -55,6 +55,7 @@ func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture { "postgres", "-c", "log_statement=all", "-c", "log_destination=stderr", + "-c", "max_connections=1000", }, }, func(config *docker.HostConfig) { // Set AutoRemove to true so that stopped container goes away From aee1f7f563b581142bf615545ce1ee07d4c09140 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Mon, 22 Jan 2024 17:54:23 +0100 Subject: [PATCH 11/21] invoices: run InvoiceStore and InvoiceRegistry tests on PostgreSQL too --- invoices/invoiceregistry_test.go | 40 +++++++++++++++++--- invoices/invoices_test.go | 65 +++++++++++++++++++++++++++----- invoices/test_utils_test.go | 2 +- 3 files changed, 91 insertions(+), 16 deletions(-) diff --git a/invoices/invoiceregistry_test.go b/invoices/invoiceregistry_test.go index a1204573c..99ad6cddd 100644 --- a/invoices/invoiceregistry_test.go +++ b/invoices/invoiceregistry_test.go @@ -116,12 +116,28 @@ func TestInvoiceRegistry(t *testing.T) { return db, testClock } - makeSQLDB := func(t *testing.T) (invpkg.InvoiceDB, *clock.TestClock) { - db := sqldb.NewTestSqliteDB(t) + // First create a shared Postgres instance so we don't spawn a new + // docker container for each test. + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + + makeSQLDB := func(t *testing.T, sqlite bool) (invpkg.InvoiceDB, + *clock.TestClock) { + + var db *sqldb.BaseDB + if sqlite { + db = sqldb.NewTestSqliteDB(t).BaseDB + } else { + db = sqldb.NewTestPostgresDB(t, pgFixture).BaseDB + } executor := sqldb.NewTransactionExecutor( db, func(tx *sql.Tx) sqldb.InvoiceQueries { - return db.BaseDB.WithTx(tx) + return db.WithTx(tx) }, ) @@ -137,8 +153,22 @@ func TestInvoiceRegistry(t *testing.T) { test.test(t, makeKeyValueDB) }) - t.Run(test.name+"_SQL", func(t *testing.T) { - test.test(t, makeSQLDB) + t.Run(test.name+"_SQLite", func(t *testing.T) { + test.test(t, + func(t *testing.T) ( + invpkg.InvoiceDB, *clock.TestClock) { + + return makeSQLDB(t, true) + }) + }) + + t.Run(test.name+"_Postgres", func(t *testing.T) { + test.test(t, + func(t *testing.T) ( + invpkg.InvoiceDB, *clock.TestClock) { + + return makeSQLDB(t, false) + }) }) } } diff --git a/invoices/invoices_test.go b/invoices/invoices_test.go index 4296f0a54..7e1baadd3 100644 --- a/invoices/invoices_test.go +++ b/invoices/invoices_test.go @@ -6,6 +6,7 @@ import ( "database/sql" "fmt" "math" + "strings" "testing" "time" @@ -36,6 +37,23 @@ var ( testNow = time.Unix(1, 0) ) +// randBytesToString will return a "safe" string from a byte slice. This is +// used to generate random strings for the invoice payment request. +func randBytesToString(buf []byte) (string, error) { + const charset = "abcdefghijklmnopqrstuvwxyz0123456789" + var stringBuilder strings.Builder + + stringBuilder.Grow(len(buf)) + for i := 0; i < len(buf); i++ { + ch := charset[int(buf[i])%len(charset)] + if err := stringBuilder.WriteByte(ch); err != nil { + return "", err + } + } + + return stringBuilder.String(), nil +} + func randInvoice(value lnwire.MilliSatoshi) (*invpkg.Invoice, error) { var ( pre lntypes.Preimage @@ -70,7 +88,11 @@ func randInvoice(value lnwire.MilliSatoshi) (*invpkg.Invoice, error) { return nil, err } if r[0]&1 == 0 { - i.PaymentRequest = r[:] + paymentReq, err := randBytesToString(r[:]) + if err != nil { + return nil, err + } + i.PaymentRequest = []byte(paymentReq) } else { i.PaymentRequest = []byte("") } @@ -200,29 +222,52 @@ func TestInvoices(t *testing.T) { return db } - makeSQLDB := func(t *testing.T) invpkg.InvoiceDB { - db := sqldb.NewTestSqliteDB(t) + // First create a shared Postgres instance so we don't spawn a new + // docker container for each test. + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + + makeSQLDB := func(t *testing.T, sqlite bool) invpkg.InvoiceDB { + var db *sqldb.BaseDB + if sqlite { + db = sqldb.NewTestSqliteDB(t).BaseDB + } else { + db = sqldb.NewTestPostgresDB(t, pgFixture).BaseDB + } executor := sqldb.NewTransactionExecutor( db, func(tx *sql.Tx) sqldb.InvoiceQueries { - return db.BaseDB.WithTx(tx) + return db.WithTx(tx) }, ) - return sqldb.NewInvoiceStore( - executor, clock.NewTestClock(testNow), - ) + testClock := clock.NewTestClock(testNow) + + return sqldb.NewInvoiceStore(executor, testClock) } for _, test := range testList { test := test - t.Run(test.name+"_KV", func(t *testing.T) { test.test(t, makeKeyValueDB) }) - t.Run(test.name+"_SQL", func(t *testing.T) { - test.test(t, makeSQLDB) + t.Run(test.name+"_SQLite", func(t *testing.T) { + test.test(t, + func(t *testing.T) invpkg.InvoiceDB { + return makeSQLDB(t, true) + }) + }) + + t.Run(test.name+"_Postgres", func(t *testing.T) { + test.test(t, + func(t *testing.T) invpkg.InvoiceDB { + return makeSQLDB(t, false) + }) }) } } diff --git a/invoices/test_utils_test.go b/invoices/test_utils_test.go index dd2638408..a0adf7dc8 100644 --- a/invoices/test_utils_test.go +++ b/invoices/test_utils_test.go @@ -231,7 +231,7 @@ func timeout() func() { go func() { select { - case <-time.After(5 * time.Second): + case <-time.After(10 * time.Second): err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) if err != nil { panic(fmt.Sprintf("error writing to std out "+ From aba45018a8021507b6daafa1ca19bc57e48174fe Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 6 Dec 2023 16:31:25 +0100 Subject: [PATCH 12/21] sqldb: merge SQLite and Postgres configs with the kvdb counterparts This commit is part of a refactor that unifies configuration of the sqldb and kvdb packages for SQL backends. In order to unify the SQLite and Postgres configuration under sqldb we first need to ensure that the final config types are compatible with the alreay deployed versions. --- sqldb/postgres.go | 102 ++++++++++++++++++++++++++------------ sqldb/postgres_fixture.go | 33 +++--------- sqldb/sqlite.go | 21 ++++---- 3 files changed, 86 insertions(+), 70 deletions(-) diff --git a/sqldb/postgres.go b/sqldb/postgres.go index f5bf79d65..cf52f7a72 100644 --- a/sqldb/postgres.go +++ b/sqldb/postgres.go @@ -6,6 +6,9 @@ import ( "database/sql" "encoding/hex" "fmt" + "net/url" + "path" + "strings" "testing" "time" @@ -15,10 +18,6 @@ import ( "github.com/stretchr/testify/require" ) -const ( - dsnTemplate = "postgres://%v:%v@%v:%d/%v?sslmode=%v" -) - var ( // DefaultPostgresFixtureLifetime is the default maximum time a Postgres // test fixture is being kept alive. After that time the docker @@ -32,31 +31,65 @@ var ( // //nolint:lll type PostgresConfig struct { - SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` - Host string `long:"host" description:"Database server hostname."` - Port int `long:"port" description:"Database server port."` - User string `long:"user" description:"Database user."` - Password string `long:"password" description:"Database user's password."` - DBName string `long:"dbname" description:"Database name to use."` - MaxOpenConnections int `long:"maxconnections" description:"Max open connections to keep alive to the database server."` - RequireSSL bool `long:"requiressl" description:"Whether to require using SSL (mode: require) when connecting to the server."` + Dsn string `long:"dsn" description:"Database connection string."` + Timeout time.Duration `long:"timeout" description:"Database connection timeout. Set to zero to disable."` + MaxConnections int `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."` + SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` } -// DSN returns the dns to connect to the database. -func (s *PostgresConfig) DSN(hidePassword bool) string { - var sslMode = "disable" - if s.RequireSSL { - sslMode = "require" +func (p *PostgresConfig) Validate() error { + if p.Dsn == "" { + return fmt.Errorf("DSN is required") } - password := s.Password - if hidePassword { - // Placeholder used for logging the DSN safely. - password = "****" + // Parse the DSN as a URL. + _, err := url.Parse(p.Dsn) + if err != nil { + return fmt.Errorf("invalid DSN: %w", err) } - return fmt.Sprintf(dsnTemplate, s.User, password, s.Host, s.Port, - s.DBName, sslMode) + return nil +} + +// replacePasswordInDSN takes a DSN string and returns it with the password +// replaced by "***". +func replacePasswordInDSN(dsn string) (string, error) { + // Parse the DSN as a URL + u, err := url.Parse(dsn) + if err != nil { + return "", err + } + + // Check if the URL has a user info part + if u.User != nil { + username := u.User.Username() + + // Reconstruct user info with "***" as password + userInfo := username + ":***@" + + // Rebuild the DSN with the modified user info + sanitizeDSN := strings.Replace( + dsn, u.User.String()+"@", userInfo, 1, + ) + + return sanitizeDSN, nil + } + + // Return the original DSN if no user info is present + return dsn, nil +} + +// getDatabaseNameFromDSN extracts the database name from a DSN string. +func getDatabaseNameFromDSN(dsn string) (string, error) { + // Parse the DSN as a URL + u, err := url.Parse(dsn) + if err != nil { + return "", err + } + + // The database name is the last segment of the path. Trim leading slash + // and return the last segment. + return path.Base(u.Path), nil } // PostgresStore is a database store implementation that uses a Postgres @@ -70,16 +103,25 @@ type PostgresStore struct { // NewPostgresStore creates a new store that is backed by a Postgres database // backend. func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { - log.Infof("Using SQL database '%s'", cfg.DSN(true)) + sanitizedDSN, err := replacePasswordInDSN(cfg.Dsn) + if err != nil { + return nil, err + } + log.Infof("Using SQL database '%s'", sanitizedDSN) - rawDB, err := sql.Open("pgx", cfg.DSN(false)) + dbName, err := getDatabaseNameFromDSN(cfg.Dsn) + if err != nil { + return nil, err + } + + rawDB, err := sql.Open("pgx", cfg.Dsn) if err != nil { return nil, err } maxConns := defaultMaxConns - if cfg.MaxOpenConnections > 0 { - maxConns = cfg.MaxOpenConnections + if cfg.MaxConnections > 0 { + maxConns = cfg.MaxConnections } rawDB.SetMaxOpenConns(maxConns) @@ -108,7 +150,7 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { }) err = applyMigrations( - postgresFS, driver, "sqlc/migrations", cfg.DBName, + postgresFS, driver, "sqlc/migrations", dbName, ) if err != nil { return nil, err @@ -149,9 +191,7 @@ func NewTestPostgresDB(t *testing.T, fixture *TestPgFixture) *PostgresStore { t.Fatal(err) } - cfg := fixture.GetConfig() - cfg.DBName = dbName - + cfg := fixture.GetConfig(dbName) store, err := NewPostgresStore(cfg) require.NoError(t, err) diff --git a/sqldb/postgres_fixture.go b/sqldb/postgres_fixture.go index 313684f96..be897657b 100644 --- a/sqldb/postgres_fixture.go +++ b/sqldb/postgres_fixture.go @@ -1,7 +1,6 @@ package sqldb import ( - "context" "database/sql" "fmt" "strconv" @@ -75,7 +74,7 @@ func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture { host: host, port: int(port), } - databaseURL := fixture.GetDSN() + databaseURL := fixture.GetConfig(testPgDBName).Dsn log.Infof("Connecting to Postgres fixture: %v\n", databaseURL) // Tell docker to hard kill the container in "expiry" seconds. @@ -104,20 +103,13 @@ func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture { return fixture } -// GetDSN returns the DSN (Data Source Name) for the started Postgres node. -func (f *TestPgFixture) GetDSN() string { - return f.GetConfig().DSN(false) -} - // GetConfig returns the full config of the Postgres node. -func (f *TestPgFixture) GetConfig() *PostgresConfig { +func (f *TestPgFixture) GetConfig(dbName string) *PostgresConfig { return &PostgresConfig{ - Host: f.host, - Port: f.port, - User: testPgUser, - Password: testPgPass, - DBName: testPgDBName, - RequireSSL: false, + Dsn: fmt.Sprintf( + "postgres://%v:%v@%v:%v/%v?sslmode=disable", + testPgUser, testPgPass, f.host, f.port, dbName, + ), } } @@ -126,16 +118,3 @@ func (f *TestPgFixture) TearDown(t *testing.T) { err := f.pool.Purge(f.resource) require.NoError(t, err, "Could not purge resource") } - -// ClearDB clears the database. -func (f *TestPgFixture) ClearDB(t *testing.T) { - dbConn, err := sql.Open("postgres", f.GetDSN()) - require.NoError(t, err) - - _, err = dbConn.ExecContext( - context.Background(), - `DROP SCHEMA IF EXISTS public CASCADE; - CREATE SCHEMA public;`, - ) - require.NoError(t, err) -} diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index 4b9895999..9b2d08487 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -40,13 +40,11 @@ const ( // //nolint:lll type SqliteConfig struct { - // SkipMigrations if true, then all the tables will be created on start - // up if they don't already exist. - SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` - - // DatabaseFileName is the full file path where the database file can be - // found. - DatabaseFileName string `long:"dbfile" description:"The full path to the database."` + Timeout time.Duration `long:"timeout" description:"The time after which a database query should be timed out."` + BusyTimeout time.Duration `long:"busytimeout" description:"The maximum amount of time to wait for a database connection to become available for a query."` + MaxConnections int `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."` + PragmaOptions []string `long:"pragmaoptions" description:"A list of pragma options to set on a database connection. For example, 'auto_vacuum=incremental'. Note that the flag must be specified multiple times if multiple options are to be set."` + SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` } // SqliteStore is a database store implementation that uses a sqlite backend. @@ -58,7 +56,7 @@ type SqliteStore struct { // NewSqliteStore attempts to open a new sqlite database based on the passed // config. -func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) { +func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { // The set of pragma options are accepted using query options. For now // we only want to ensure that foreign key constraints are properly // enforced. @@ -107,7 +105,7 @@ func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) { // details on the formatting here, see the modernc.org/sqlite docs: // https://pkg.go.dev/modernc.org/sqlite#Driver.Open. dsn := fmt.Sprintf( - "%v?%v&%v", cfg.DatabaseFileName, sqliteOptions.Encode(), + "%v?%v&%v", dbPath, sqliteOptions.Encode(), sqliteTxLockImmediate, ) db, err := sql.Open("sqlite", dsn) @@ -171,9 +169,8 @@ func NewTestSqliteDB(t *testing.T) *SqliteStore { // an in mem version to speed up tests dbFileName := filepath.Join(t.TempDir(), "tmp.db") sqlDB, err := NewSqliteStore(&SqliteConfig{ - DatabaseFileName: dbFileName, - SkipMigrations: false, - }) + SkipMigrations: false, + }, dbFileName) require.NoError(t, err) t.Cleanup(func() { From debfaf41b9faa12c9cd38d21e8ea36d62e9c0a5a Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 6 Dec 2023 17:00:59 +0100 Subject: [PATCH 13/21] lncfg: use the sqldb config types instead of the kvdb counterparts --- config_builder.go | 3 ++- lncfg/db.go | 67 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/config_builder.go b/config_builder.go index 27e9f47f6..3f9dd7c27 100644 --- a/config_builder.go +++ b/config_builder.go @@ -1298,8 +1298,9 @@ func initNeutrinoBackend(ctx context.Context, cfg *Config, chainDir string, ) switch { case cfg.DB.Backend == kvdb.SqliteBackendName: + sqliteConfig := lncfg.GetSqliteConfigKVDB(cfg.DB.Sqlite) db, err = kvdb.Open( - kvdb.SqliteBackendName, ctx, cfg.DB.Sqlite, dbPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, dbPath, lncfg.SqliteNeutrinoDBName, lncfg.NSNeutrinoDB, ) diff --git a/lncfg/db.go b/lncfg/db.go index d1ecf6386..d88feced7 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -14,6 +14,7 @@ import ( "github.com/lightningnetwork/lnd/kvdb/sqlite" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lnwallet/btcwallet" + "github.com/lightningnetwork/lnd/sqldb" ) const ( @@ -78,9 +79,9 @@ type DB struct { Bolt *kvdb.BoltConfig `group:"bolt" namespace:"bolt" description:"Bolt settings."` - Postgres *postgres.Config `group:"postgres" namespace:"postgres" description:"Postgres settings."` + Postgres *sqldb.PostgresConfig `group:"postgres" namespace:"postgres" description:"Postgres settings."` - Sqlite *sqlite.Config `group:"sqlite" namespace:"sqlite" description:"Sqlite settings."` + Sqlite *sqldb.SqliteConfig `group:"sqlite" namespace:"sqlite" description:"Sqlite settings."` NoGraphCache bool `long:"no-graph-cache" description:"Don't use the in-memory graph cache for path finding. Much slower but uses less RAM. Can only be used with a bolt database backend."` @@ -103,10 +104,10 @@ func DefaultDB() *DB { // Allow at most 32 MiB messages by default. MaxMsgSize: 32768 * 1024, }, - Postgres: &postgres.Config{ + Postgres: &sqldb.PostgresConfig{ MaxConnections: defaultPostgresMaxConnections, }, - Sqlite: &sqlite.Config{ + Sqlite: &sqldb.SqliteConfig{ MaxConnections: defaultSqliteMaxConnections, BusyTimeout: defaultSqliteBusyTimeout, }, @@ -118,8 +119,8 @@ func (db *DB) Validate() error { switch db.Backend { case BoltBackend, SqliteBackend: case PostgresBackend: - if db.Postgres.Dsn == "" { - return fmt.Errorf("postgres dsn must be set") + if err := db.Postgres.Validate(); err != nil { + return err } case EtcdBackend: @@ -223,6 +224,26 @@ type DatabaseBackends struct { CloseFuncs map[string]func() error } +// GetPostgresConfigKVDB converts a sqldb.PostgresConfig to a kvdb +// postgres.Config. +func GetPostgresConfigKVDB(cfg *sqldb.PostgresConfig) *postgres.Config { + return &postgres.Config{ + Dsn: cfg.Dsn, + Timeout: cfg.Timeout, + MaxConnections: cfg.MaxConnections, + } +} + +// GetSqliteConfigKVDB converts a sqldb.SqliteConfig to a kvdb sqlite.Config. +func GetSqliteConfigKVDB(cfg *sqldb.SqliteConfig) *sqlite.Config { + return &sqlite.Config{ + Timeout: cfg.Timeout, + BusyTimeout: cfg.BusyTimeout, + MaxConnections: cfg.MaxConnections, + PragmaOptions: cfg.PragmaOptions, + } +} + // GetBackends returns a set of kvdb.Backends as set in the DB config. func (db *DB) GetBackends(ctx context.Context, chanDBPath, walletDBPath, towerServerDBPath string, towerClientEnabled, @@ -342,9 +363,14 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, }, nil case PostgresBackend: + // Convert the sqldb PostgresConfig to a kvdb postgres.Config. + // This is a temporary measure until we migrate all kvdb SQL + // users to native SQL. + postgresConfig := GetPostgresConfigKVDB(db.Postgres) + postgresBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSChannelDB, + postgresConfig, NSChannelDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres graph "+ @@ -354,7 +380,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, postgresMacaroonBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSMacaroonDB, + postgresConfig, NSMacaroonDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres "+ @@ -364,7 +390,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, postgresDecayedLogBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSDecayedLogDB, + postgresConfig, NSDecayedLogDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres "+ @@ -374,7 +400,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, postgresTowerClientBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSTowerClientDB, + postgresConfig, NSTowerClientDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres tower "+ @@ -384,7 +410,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, postgresTowerServerBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSTowerServerDB, + postgresConfig, NSTowerServerDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres tower "+ @@ -394,7 +420,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, postgresWalletBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSWalletDB, + postgresConfig, NSWalletDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres macaroon "+ @@ -434,6 +460,11 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, }, nil case SqliteBackend: + // Convert the sqldb SqliteConfig to a kvdb sqlite.Config. + // This is a temporary measure until we migrate all kvdb SQL + // users to native SQL. + sqliteConfig := GetSqliteConfigKVDB(db.Sqlite) + // Note that for sqlite, we put kv tables for the channel.db, // wtclient.db and sphinxreplay.db all in the channel.sqlite db. // The tables for wallet.db and macaroon.db are in the @@ -445,7 +476,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, // would cause deadlocks in the code due to the wallet db often // being accessed during a write to another db. sqliteBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, chanDBPath, SqliteChannelDBName, NSChannelDB, ) if err != nil { @@ -455,7 +486,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, closeFuncs[NSChannelDB] = sqliteBackend.Close sqliteMacaroonBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, walletDBPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, walletDBPath, SqliteChainDBName, NSMacaroonDB, ) if err != nil { @@ -465,7 +496,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, closeFuncs[NSMacaroonDB] = sqliteMacaroonBackend.Close sqliteDecayedLogBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, chanDBPath, SqliteChannelDBName, NSDecayedLogDB, ) if err != nil { @@ -475,7 +506,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, closeFuncs[NSDecayedLogDB] = sqliteDecayedLogBackend.Close sqliteTowerClientBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, chanDBPath, SqliteChannelDBName, NSTowerClientDB, ) if err != nil { @@ -485,7 +516,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, closeFuncs[NSTowerClientDB] = sqliteTowerClientBackend.Close sqliteTowerServerBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, + kvdb.SqliteBackendName, ctx, sqliteConfig, towerServerDBPath, SqliteTowerDBName, NSTowerServerDB, ) if err != nil { @@ -495,7 +526,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, closeFuncs[NSTowerServerDB] = sqliteTowerServerBackend.Close sqliteWalletBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, walletDBPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, walletDBPath, SqliteChainDBName, NSWalletDB, ) if err != nil { From e31879ea0aaf186c05bbf2e68407fc118b42f702 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 6 Dec 2023 17:15:15 +0100 Subject: [PATCH 14/21] lncfg: instantiate the native SQL backend if configured This commit adds an optional separate "native" SQL backend that will host all tables supporting native SQL. For now since we don't have many such tables we'll keep it in one file for SQLite, but it may be split up if desired. --- lncfg/db.go | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/lncfg/db.go b/lncfg/db.go index d88feced7..49f506bb1 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -3,6 +3,7 @@ package lncfg import ( "context" "fmt" + "path" "path/filepath" "time" @@ -29,6 +30,7 @@ const ( SqliteChainDBName = "chain.sqlite" SqliteNeutrinoDBName = "neutrino.sqlite" SqliteTowerDBName = "watchtower.sqlite" + SqliteNativeDBName = "lnd.sqlite" BoltBackend = "bolt" EtcdBackend = "etcd" @@ -83,6 +85,8 @@ type DB struct { Sqlite *sqldb.SqliteConfig `group:"sqlite" namespace:"sqlite" description:"Sqlite settings."` + UseNativeSQL bool `long:"use-native-sql" description:"Use native SQL for tables that already support it."` + NoGraphCache bool `long:"no-graph-cache" description:"Don't use the in-memory graph cache for path finding. Much slower but uses less RAM. Can only be used with a bolt database backend."` PruneRevocation bool `long:"prune-revocation" description:"Run the optional migration that prunes the revocation logs to save disk space."` @@ -111,6 +115,7 @@ func DefaultDB() *DB { MaxConnections: defaultSqliteMaxConnections, BusyTimeout: defaultSqliteBusyTimeout, }, + UseNativeSQL: false, } } @@ -215,6 +220,11 @@ type DatabaseBackends struct { // the underlying wallet database from. WalletDB btcwallet.LoaderOption + // NativeSQLStore is a pointer to a native SQL store that can be used + // for native SQL queries for tables that already support it. This may + // be nil if the use-native-sql flag was not set. + NativeSQLStore *sqldb.BaseDB + // Remote indicates whether the database backends are remote, possibly // replicated instances or local bbolt or sqlite backed databases. Remote bool @@ -428,6 +438,20 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, } closeFuncs[NSWalletDB] = postgresWalletBackend.Close + var nativeSQLStore *sqldb.BaseDB + if db.UseNativeSQL { + nativePostgresStore, err := sqldb.NewPostgresStore( + db.Postgres, + ) + if err != nil { + return nil, fmt.Errorf("error opening "+ + "native postgres store: %v", err) + } + + nativeSQLStore = nativePostgresStore.BaseDB + closeFuncs[PostgresBackend] = nativePostgresStore.Close + } + // Warn if the user is trying to switch over to a Postgres DB // while there is a wallet or channel bbolt DB still present. warnExistingBoltDBs( @@ -455,8 +479,9 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, WalletDB: btcwallet.LoaderWithExternalWalletDB( postgresWalletBackend, ), - Remote: true, - CloseFuncs: closeFuncs, + NativeSQLStore: nativeSQLStore, + Remote: true, + CloseFuncs: closeFuncs, }, nil case SqliteBackend: @@ -535,6 +560,21 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, } closeFuncs[NSWalletDB] = sqliteWalletBackend.Close + var nativeSQLStore *sqldb.BaseDB + if db.UseNativeSQL { + nativeSQLiteStore, err := sqldb.NewSqliteStore( + db.Sqlite, + path.Join(chanDBPath, SqliteNativeDBName), + ) + if err != nil { + return nil, fmt.Errorf("error opening "+ + "native SQLite store: %v", err) + } + + nativeSQLStore = nativeSQLiteStore.BaseDB + closeFuncs[SqliteBackend] = nativeSQLiteStore.Close + } + // Warn if the user is trying to switch over to a sqlite DB // while there is a wallet or channel bbolt DB still present. warnExistingBoltDBs( @@ -562,7 +602,8 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, WalletDB: btcwallet.LoaderWithExternalWalletDB( sqliteWalletBackend, ), - CloseFuncs: closeFuncs, + NativeSQLStore: nativeSQLStore, + CloseFuncs: closeFuncs, }, nil } From 88370de1ab740bcaa13bb6ca0cb7e3ff8b4c7854 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 6 Dec 2023 17:23:16 +0100 Subject: [PATCH 15/21] lnd: use native SQL invoice store if configured --- config_builder.go | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/config_builder.go b/config_builder.go index 3f9dd7c27..5195777e1 100644 --- a/config_builder.go +++ b/config_builder.go @@ -3,6 +3,7 @@ package lnd import ( "bytes" "context" + "database/sql" "errors" "fmt" "io/ioutil" @@ -31,6 +32,7 @@ import ( "github.com/lightningnetwork/lnd/chainntnfs" "github.com/lightningnetwork/lnd/chainreg" "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/invoices" "github.com/lightningnetwork/lnd/keychain" "github.com/lightningnetwork/lnd/kvdb" @@ -42,6 +44,7 @@ import ( "github.com/lightningnetwork/lnd/macaroons" "github.com/lightningnetwork/lnd/rpcperms" "github.com/lightningnetwork/lnd/signal" + "github.com/lightningnetwork/lnd/sqldb" "github.com/lightningnetwork/lnd/walletunlocker" "github.com/lightningnetwork/lnd/watchtower" "github.com/lightningnetwork/lnd/watchtower/wtclient" @@ -869,6 +872,11 @@ type DatabaseInstances struct { // WalletDB is the configuration for loading the wallet database using // the btcwallet's loader. WalletDB btcwallet.LoaderOption + + // NativeSQLStore is a pointer to a native SQL store that can be used + // for native SQL queries for tables that already support it. This may + // be nil if the use-native-sql flag was not set. + NativeSQLStore *sqldb.BaseDB } // DefaultDatabaseBuilder is a type that builds the default database backends @@ -923,10 +931,11 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( // state DB point to the same local or remote DB and the same namespace // within that DB. dbs := &DatabaseInstances{ - HeightHintDB: databaseBackends.HeightHintDB, - MacaroonDB: databaseBackends.MacaroonDB, - DecayedLogDB: databaseBackends.DecayedLogDB, - WalletDB: databaseBackends.WalletDB, + HeightHintDB: databaseBackends.HeightHintDB, + MacaroonDB: databaseBackends.MacaroonDB, + DecayedLogDB: databaseBackends.DecayedLogDB, + WalletDB: databaseBackends.WalletDB, + NativeSQLStore: databaseBackends.NativeSQLStore, } cleanUp := func() { // We can just close the returned close functions directly. Even @@ -1011,11 +1020,21 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( // using the same struct (and DB backend) instance. dbs.ChanStateDB = dbs.GraphDB - // For now the only InvoiceDB implementation is the *channeldb.DB. - // - // TODO(positiveblue): use a sql first implementation for this - // interface. - dbs.InvoiceDB = dbs.GraphDB + // Instantiate a native SQL invoice store if the flag is set. + if d.cfg.DB.UseNativeSQL { + executor := sqldb.NewTransactionExecutor( + dbs.NativeSQLStore, + func(tx *sql.Tx) sqldb.InvoiceQueries { + return dbs.NativeSQLStore.WithTx(tx) + }, + ) + + dbs.InvoiceDB = sqldb.NewInvoiceStore( + executor, clock.NewDefaultClock(), + ) + } else { + dbs.InvoiceDB = dbs.GraphDB + } // Wrap the watchtower client DB and make sure we clean up. if cfg.WtClient.Active { From ce1b57da2db1617579401c28eae6d23e08b196ac Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 6 Dec 2023 19:47:56 +0100 Subject: [PATCH 16/21] sqldb: exclude sqlite from the JS and unsupported platform builds --- sqldb/config.go | 55 +++++++++++++++++++++++ sqldb/no_sqlite.go | 18 ++++++++ sqldb/postgres.go | 60 ------------------------- sqldb/postgres_fixture.go | 35 +++++++++++++++ sqldb/sqlerrors.go | 2 + sqldb/sqlerrors_no_sqlite.go | 86 ++++++++++++++++++++++++++++++++++++ sqldb/sqlite.go | 25 +---------- 7 files changed, 198 insertions(+), 83 deletions(-) create mode 100644 sqldb/config.go create mode 100644 sqldb/no_sqlite.go create mode 100644 sqldb/sqlerrors_no_sqlite.go diff --git a/sqldb/config.go b/sqldb/config.go new file mode 100644 index 000000000..5bc807806 --- /dev/null +++ b/sqldb/config.go @@ -0,0 +1,55 @@ +package sqldb + +import ( + "fmt" + "net/url" + "time" +) + +const ( + // defaultMaxConns is the number of permitted active and idle + // connections. We want to limit this so it isn't unlimited. We use the + // same value for the number of idle connections as, this can speed up + // queries given a new connection doesn't need to be established each + // time. + defaultMaxConns = 25 + + // connIdleLifetime is the amount of time a connection can be idle. + connIdleLifetime = 5 * time.Minute +) + +// SqliteConfig holds all the config arguments needed to interact with our +// sqlite DB. +// +//nolint:lll +type SqliteConfig struct { + Timeout time.Duration `long:"timeout" description:"The time after which a database query should be timed out."` + BusyTimeout time.Duration `long:"busytimeout" description:"The maximum amount of time to wait for a database connection to become available for a query."` + MaxConnections int `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."` + PragmaOptions []string `long:"pragmaoptions" description:"A list of pragma options to set on a database connection. For example, 'auto_vacuum=incremental'. Note that the flag must be specified multiple times if multiple options are to be set."` + SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` +} + +// PostgresConfig holds the postgres database configuration. +// +//nolint:lll +type PostgresConfig struct { + Dsn string `long:"dsn" description:"Database connection string."` + Timeout time.Duration `long:"timeout" description:"Database connection timeout. Set to zero to disable."` + MaxConnections int `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."` + SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` +} + +func (p *PostgresConfig) Validate() error { + if p.Dsn == "" { + return fmt.Errorf("DSN is required") + } + + // Parse the DSN as a URL. + _, err := url.Parse(p.Dsn) + if err != nil { + return fmt.Errorf("invalid DSN: %w", err) + } + + return nil +} diff --git a/sqldb/no_sqlite.go b/sqldb/no_sqlite.go new file mode 100644 index 000000000..9ea35c43c --- /dev/null +++ b/sqldb/no_sqlite.go @@ -0,0 +1,18 @@ +//go:build js || (windows && (arm || 386)) || (linux && (ppc64 || mips || mipsle || mips64)) + +package sqldb + +import "fmt" + +// SqliteStore is a database store implementation that uses a sqlite backend. +type SqliteStore struct { + cfg *SqliteConfig + + *BaseDB +} + +// NewSqliteStore attempts to open a new sqlite database based on the passed +// config. +func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { + return nil, fmt.Errorf("SQLite backend not supported in WebAssembly") +} diff --git a/sqldb/postgres.go b/sqldb/postgres.go index cf52f7a72..e6e88c93b 100644 --- a/sqldb/postgres.go +++ b/sqldb/postgres.go @@ -1,21 +1,15 @@ package sqldb import ( - "context" - "crypto/rand" "database/sql" - "encoding/hex" - "fmt" "net/url" "path" "strings" - "testing" "time" postgres_migrate "github.com/golang-migrate/migrate/v4/database/postgres" _ "github.com/golang-migrate/migrate/v4/source/file" // Read migrations from files. // nolint:lll "github.com/lightningnetwork/lnd/sqldb/sqlc" - "github.com/stretchr/testify/require" ) var ( @@ -27,30 +21,6 @@ var ( DefaultPostgresFixtureLifetime = 10 * time.Minute ) -// PostgresConfig holds the postgres database configuration. -// -//nolint:lll -type PostgresConfig struct { - Dsn string `long:"dsn" description:"Database connection string."` - Timeout time.Duration `long:"timeout" description:"Database connection timeout. Set to zero to disable."` - MaxConnections int `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."` - SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` -} - -func (p *PostgresConfig) Validate() error { - if p.Dsn == "" { - return fmt.Errorf("DSN is required") - } - - // Parse the DSN as a URL. - _, err := url.Parse(p.Dsn) - if err != nil { - return fmt.Errorf("invalid DSN: %w", err) - } - - return nil -} - // replacePasswordInDSN takes a DSN string and returns it with the password // replaced by "***". func replacePasswordInDSN(dsn string) (string, error) { @@ -167,33 +137,3 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { }, }, nil } - -// NewTestPostgresDB is a helper function that creates a Postgres database for -// testing using the given fixture. -func NewTestPostgresDB(t *testing.T, fixture *TestPgFixture) *PostgresStore { - t.Helper() - - // Create random database name. - randBytes := make([]byte, 8) - _, err := rand.Read(randBytes) - if err != nil { - t.Fatal(err) - } - - dbName := "test_" + hex.EncodeToString(randBytes) - - t.Logf("Creating new Postgres DB '%s' for testing", dbName) - - _, err = fixture.db.ExecContext( - context.Background(), "CREATE DATABASE "+dbName, - ) - if err != nil { - t.Fatal(err) - } - - cfg := fixture.GetConfig(dbName) - store, err := NewPostgresStore(cfg) - require.NoError(t, err) - - return store -} diff --git a/sqldb/postgres_fixture.go b/sqldb/postgres_fixture.go index be897657b..a0f0893ce 100644 --- a/sqldb/postgres_fixture.go +++ b/sqldb/postgres_fixture.go @@ -1,7 +1,12 @@ +//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) && !(netbsd || openbsd) + package sqldb import ( + "context" + "crypto/rand" "database/sql" + "encoding/hex" "fmt" "strconv" "strings" @@ -118,3 +123,33 @@ func (f *TestPgFixture) TearDown(t *testing.T) { err := f.pool.Purge(f.resource) require.NoError(t, err, "Could not purge resource") } + +// NewTestPostgresDB is a helper function that creates a Postgres database for +// testing using the given fixture. +func NewTestPostgresDB(t *testing.T, fixture *TestPgFixture) *PostgresStore { + t.Helper() + + // Create random database name. + randBytes := make([]byte, 8) + _, err := rand.Read(randBytes) + if err != nil { + t.Fatal(err) + } + + dbName := "test_" + hex.EncodeToString(randBytes) + + t.Logf("Creating new Postgres DB '%s' for testing", dbName) + + _, err = fixture.db.ExecContext( + context.Background(), "CREATE DATABASE "+dbName, + ) + if err != nil { + t.Fatal(err) + } + + cfg := fixture.GetConfig(dbName) + store, err := NewPostgresStore(cfg) + require.NoError(t, err) + + return store +} diff --git a/sqldb/sqlerrors.go b/sqldb/sqlerrors.go index 930347676..a939ec499 100644 --- a/sqldb/sqlerrors.go +++ b/sqldb/sqlerrors.go @@ -1,3 +1,5 @@ +//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) + package sqldb import ( diff --git a/sqldb/sqlerrors_no_sqlite.go b/sqldb/sqlerrors_no_sqlite.go new file mode 100644 index 000000000..717d94152 --- /dev/null +++ b/sqldb/sqlerrors_no_sqlite.go @@ -0,0 +1,86 @@ +//go:build js || (windows && (arm || 386)) || (linux && (ppc64 || mips || mipsle || mips64)) + +package sqldb + +import ( + "errors" + "fmt" + + "github.com/jackc/pgconn" + "github.com/jackc/pgerrcode" +) + +var ( + // ErrRetriesExceeded is returned when a transaction is retried more + // than the max allowed valued without a success. + ErrRetriesExceeded = errors.New("db tx retries exceeded") +) + +// MapSQLError attempts to interpret a given error as a database agnostic SQL +// error. +func MapSQLError(err error) error { + // Attempt to interpret the error as a postgres error. + var pqErr *pgconn.PgError + if errors.As(err, &pqErr) { + return parsePostgresError(pqErr) + } + + // Return original error if it could not be classified as a database + // specific error. + return err +} + +// parsePostgresError attempts to parse a postgres error as a database agnostic +// SQL error. +func parsePostgresError(pqErr *pgconn.PgError) error { + switch pqErr.Code { + // Handle unique constraint violation error. + case pgerrcode.UniqueViolation: + return &ErrSQLUniqueConstraintViolation{ + DBError: pqErr, + } + + // Unable to serialize the transaction, so we'll need to try again. + case pgerrcode.SerializationFailure: + return &ErrSerializationError{ + DBError: pqErr, + } + + default: + return fmt.Errorf("unknown postgres error: %w", pqErr) + } +} + +// ErrSQLUniqueConstraintViolation is an error type which represents a database +// agnostic SQL unique constraint violation. +type ErrSQLUniqueConstraintViolation struct { + DBError error +} + +func (e ErrSQLUniqueConstraintViolation) Error() string { + return fmt.Sprintf("sql unique constraint violation: %v", e.DBError) +} + +// ErrSerializationError is an error type which represents a database agnostic +// error that a transaction couldn't be serialized with other concurrent db +// transactions. +type ErrSerializationError struct { + DBError error +} + +// Unwrap returns the wrapped error. +func (e ErrSerializationError) Unwrap() error { + return e.DBError +} + +// Error returns the error message. +func (e ErrSerializationError) Error() string { + return e.DBError.Error() +} + +// IsSerializationError returns true if the given error is a serialization +// error. +func IsSerializationError(err error) bool { + var serializationError *ErrSerializationError + return errors.As(err, &serializationError) +} diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index 9b2d08487..705d5cc47 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -1,3 +1,5 @@ +//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) + package sqldb import ( @@ -6,7 +8,6 @@ import ( "net/url" "path/filepath" "testing" - "time" sqlite_migrate "github.com/golang-migrate/migrate/v4/database/sqlite" "github.com/lightningnetwork/lnd/sqldb/sqlc" @@ -23,30 +24,8 @@ const ( // sqliteTxLockImmediate is a dsn option used to ensure that write // transactions are started immediately. sqliteTxLockImmediate = "_txlock=immediate" - - // defaultMaxConns is the number of permitted active and idle - // connections. We want to limit this so it isn't unlimited. We use the - // same value for the number of idle connections as, this can speed up - // queries given a new connection doesn't need to be established each - // time. - defaultMaxConns = 25 - - // connIdleLifetime is the amount of time a connection can be idle. - connIdleLifetime = 5 * time.Minute ) -// SqliteConfig holds all the config arguments needed to interact with our -// sqlite DB. -// -//nolint:lll -type SqliteConfig struct { - Timeout time.Duration `long:"timeout" description:"The time after which a database query should be timed out."` - BusyTimeout time.Duration `long:"busytimeout" description:"The maximum amount of time to wait for a database connection to become available for a query."` - MaxConnections int `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."` - PragmaOptions []string `long:"pragmaoptions" description:"A list of pragma options to set on a database connection. For example, 'auto_vacuum=incremental'. Note that the flag must be specified multiple times if multiple options are to be set."` - SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` -} - // SqliteStore is a database store implementation that uses a sqlite backend. type SqliteStore struct { cfg *SqliteConfig From 42508d44e57752ea776b08a664dfadaff503f6e3 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 6 Dec 2023 19:50:24 +0100 Subject: [PATCH 17/21] lnd: update sample-lnd.conf --- sample-lnd.conf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sample-lnd.conf b/sample-lnd.conf index 2e7f79376..6447e717d 100644 --- a/sample-lnd.conf +++ b/sample-lnd.conf @@ -1306,6 +1306,11 @@ ; the future. ; db.no-rev-log-amt-data=false +; If set to true, native SQL will be used instead of KV emulation for tables +; that support it already. Note: this is an experimental feature, use at your +; own risk. +; db.use-native-sql=false + [etcd] @@ -1401,6 +1406,9 @@ ; Example: ; db.postgres.maxconnections= +; Whether to skip executing schema migrations. +; db.postgres.skipmigrations=false + [sqlite] @@ -1425,6 +1433,8 @@ ; db.sqlite.pragmaoptions=auto_vacuum=incremental ; db.sqlite.pragmaoptions=temp_store=MEMORY +; Whether to skip executing schema migrations. +; db.sqlite.skipmigrations=false [bolt] From 7a45bbbeabca7c5f8820ac2a558b772a3c551f52 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Thu, 7 Dec 2023 21:25:19 +0100 Subject: [PATCH 18/21] lnd: call into the correct InvoiceStore from RPC calls --- rpcserver.go | 2 +- server.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/rpcserver.go b/rpcserver.go index fdfb8ebd6..d55c97257 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -5806,7 +5806,7 @@ func (r *rpcServer) ListInvoices(ctx context.Context, CreationDateEnd: int64(req.CreationDateEnd), } - invoiceSlice, err := r.server.miscDB.QueryInvoices(ctx, q) + invoiceSlice, err := r.server.invoicesDB.QueryInvoices(ctx, q) if err != nil { return nil, fmt.Errorf("unable to query invoices: %w", err) } diff --git a/server.go b/server.go index cb1540a2c..32dbc9cf0 100644 --- a/server.go +++ b/server.go @@ -246,6 +246,8 @@ type server struct { // channel DB that haven't been separated out yet. miscDB *channeldb.DB + invoicesDB invoices.InvoiceDB + aliasMgr *aliasmgr.Manager htlcSwitch *htlcswitch.Switch @@ -564,6 +566,7 @@ func newServer(cfg *Config, listenAddrs []net.Addr, chanStateDB: dbs.ChanStateDB.ChannelStateDB(), addrSource: dbs.ChanStateDB, miscDB: dbs.ChanStateDB, + invoicesDB: dbs.InvoiceDB, cc: cc, sigPool: lnwallet.NewSigPool(cfg.Workers.Sig, cc.Signer), writePool: writePool, From ba8e7550d5f2fa522ddbb7d135aabf7e2f26c530 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 12 Dec 2023 14:50:15 +0100 Subject: [PATCH 19/21] itest: add the -nativesql flag to run SQL itests with native SQL tables --- itest/lnd_test.go | 5 ++++- lntest/harness.go | 4 ++-- lntest/harness_node_manager.go | 10 ++++++++-- lntest/harness_setup.go | 4 ++-- lntest/node/config.go | 7 +++++++ make/testing_flags.mk | 6 ++++++ 6 files changed, 29 insertions(+), 7 deletions(-) diff --git a/itest/lnd_test.go b/itest/lnd_test.go index cb9f60dbf..9c823b5a8 100644 --- a/itest/lnd_test.go +++ b/itest/lnd_test.go @@ -71,6 +71,9 @@ var ( dbBackendFlag = flag.String("dbbackend", "bbolt", "Database backend "+ "(bbolt, etcd, postgres)") + nativeSQLFlag = flag.Bool("nativesql", false, "Database backend to "+ + "use native SQL when applicable (only for sqlite and postgres") + // lndExecutable is the full path to the lnd binary. lndExecutable = flag.String( "lndexec", itestLndBinary, "full path to lnd binary", @@ -95,7 +98,7 @@ func TestLightningNetworkDaemon(t *testing.T) { // Get the binary path and setup the harness test. binary := getLndBinary(t) harnessTest := lntest.SetupHarness( - t, binary, *dbBackendFlag, feeService, + t, binary, *dbBackendFlag, *nativeSQLFlag, feeService, ) defer harnessTest.Stop() diff --git a/lntest/harness.go b/lntest/harness.go index 23f5a8b43..c4996a730 100644 --- a/lntest/harness.go +++ b/lntest/harness.go @@ -103,14 +103,14 @@ type HarnessTest struct { // NewHarnessTest creates a new instance of a harnessTest from a regular // testing.T instance. func NewHarnessTest(t *testing.T, lndBinary string, feeService WebFeeService, - dbBackend node.DatabaseBackend) *HarnessTest { + dbBackend node.DatabaseBackend, nativeSQL bool) *HarnessTest { t.Helper() // Create the run context. ctxt, cancel := context.WithCancel(context.Background()) - manager := newNodeManager(lndBinary, dbBackend) + manager := newNodeManager(lndBinary, dbBackend, nativeSQL) return &HarnessTest{ T: t, diff --git a/lntest/harness_node_manager.go b/lntest/harness_node_manager.go index dd1b59ea3..3b0c412d9 100644 --- a/lntest/harness_node_manager.go +++ b/lntest/harness_node_manager.go @@ -31,6 +31,10 @@ type nodeManager struct { // dbBackend sets the database backend to use. dbBackend node.DatabaseBackend + // nativeSQL sets the database backend to use native SQL when + // applicable. + nativeSQL bool + // activeNodes is a map of all running nodes, format: // {pubkey: *HarnessNode}. activeNodes map[uint32]*node.HarnessNode @@ -48,12 +52,13 @@ type nodeManager struct { } // newNodeManager creates a new node manager instance. -func newNodeManager(lndBinary string, - dbBackend node.DatabaseBackend) *nodeManager { +func newNodeManager(lndBinary string, dbBackend node.DatabaseBackend, + nativeSQL bool) *nodeManager { return &nodeManager{ lndBinary: lndBinary, dbBackend: dbBackend, + nativeSQL: nativeSQL, activeNodes: make(map[uint32]*node.HarnessNode), standbyNodes: make(map[uint32]*node.HarnessNode), } @@ -80,6 +85,7 @@ func (nm *nodeManager) newNode(t *testing.T, name string, extraArgs []string, ExtraArgs: extraArgs, FeeURL: nm.feeServiceURL, DBBackend: nm.dbBackend, + NativeSQL: nm.nativeSQL, NodeID: nm.nextNodeID(), LndBinary: nm.lndBinary, NetParams: harnessNetParams, diff --git a/lntest/harness_setup.go b/lntest/harness_setup.go index 20442f9c2..61eac5d2f 100644 --- a/lntest/harness_setup.go +++ b/lntest/harness_setup.go @@ -18,7 +18,7 @@ import ( // 4. connect the miner and the chain backend. // 5. start the HarnessTest. func SetupHarness(t *testing.T, binaryPath, dbBackendName string, - feeService WebFeeService) *HarnessTest { + nativeSQL bool, feeService WebFeeService) *HarnessTest { t.Log("Setting up HarnessTest...") @@ -30,7 +30,7 @@ func SetupHarness(t *testing.T, binaryPath, dbBackendName string, dbBackend := prepareDBBackend(t, dbBackendName) // Create a new HarnessTest. - ht := NewHarnessTest(t, binaryPath, feeService, dbBackend) + ht := NewHarnessTest(t, binaryPath, feeService, dbBackend, nativeSQL) // Init the miner. t.Log("Prepare the miner and mine blocks to activate segwit...") diff --git a/lntest/node/config.go b/lntest/node/config.go index 449ad2fad..1573af42e 100644 --- a/lntest/node/config.go +++ b/lntest/node/config.go @@ -119,6 +119,7 @@ type BaseNodeConfig struct { DBBackend DatabaseBackend PostgresDsn string + NativeSQL bool // NodeID is a unique ID used to identify the node. NodeID uint32 @@ -277,11 +278,17 @@ func (cfg *BaseNodeConfig) GenArgs() []string { case BackendPostgres: args = append(args, "--db.backend=postgres") args = append(args, "--db.postgres.dsn="+cfg.PostgresDsn) + if cfg.NativeSQL { + args = append(args, "--db.use-native-sql") + } case BackendSqlite: args = append(args, "--db.backend=sqlite") args = append(args, fmt.Sprintf("--db.sqlite.busytimeout=%v", wait.SqliteBusyTimeout)) + if cfg.NativeSQL { + args = append(args, "--db.use-native-sql") + } } if cfg.FeeURL != "" { diff --git a/make/testing_flags.mk b/make/testing_flags.mk index 68afc9aa0..df20cdb7e 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -60,10 +60,16 @@ DEV_TAGS += kvdb_etcd endif ifeq ($(dbbackend),postgres) +ifneq ($(nativesql),) +ITEST_FLAGS += -nativesql +endif DEV_TAGS += kvdb_postgres endif ifeq ($(dbbackend),sqlite) +ifneq ($(nativesql),) +ITEST_FLAGS += -nativesql +endif DEV_TAGS += kvdb_sqlite endif From 29ae17ba71c546b71ed66d5dfec9464773a5c04c Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 12 Dec 2023 14:52:14 +0100 Subject: [PATCH 20/21] github: add nativesql itests to the matrix --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d51ca9c32..deceba636 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -252,6 +252,10 @@ jobs: args: backend=bitcoind dbbackend=postgres - name: bitcoind-sqlite args: backend=bitcoind dbbackend=sqlite + - name: bitcoind-postgres-nativesql + args: backend=bitcoind dbbackend=postgres nativesql=true + - name: bitcoind-sqlite-nativesql + args: backend=bitcoind dbbackend=sqlite nativesql=true - name: neutrino args: backend=neutrino steps: From 6c09617568a00de00265eb93143dac648358f4d8 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 23 Jan 2024 17:53:19 +0100 Subject: [PATCH 21/21] docs: update release notes for v0.18.0-beta --- docs/release-notes/release-notes-0.18.0.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/release-notes/release-notes-0.18.0.md b/docs/release-notes/release-notes-0.18.0.md index 4ef9d7dd9..4af62093f 100644 --- a/docs/release-notes/release-notes-0.18.0.md +++ b/docs/release-notes/release-notes-0.18.0.md @@ -327,6 +327,10 @@ * [Refactor UpdateInvoice](https://github.com/lightningnetwork/lnd/pull/8100) to make it simpler to adjust code to also support SQL InvoiceDB implementation. +* [InvoiceDB implementation](https://github.com/lightningnetwork/lnd/pull/8052) + for SQL backends enabling new users to optionally use an experimental native + SQL invoices database. + ## Code Health * [Remove database pointers](https://github.com/lightningnetwork/lnd/pull/8117)