From 2596d34619cd194f714f14729b8b438e64dd44d4 Mon Sep 17 00:00:00 2001 From: ziggie Date: Sat, 10 Jan 2026 23:38:25 +0100 Subject: [PATCH 01/16] paymentsdb+sqldb: add migration related query Add a migration specific query which allows to set the failure reason when inserting a payment into the db. --- payments/db/sql_store.go | 19 ++++++ sqldb/sqlc/payments.sql.go | 111 ++++++++++++++++++++++++++++++++ sqldb/sqlc/querier.go | 8 +++ sqldb/sqlc/queries/payments.sql | 41 ++++++++++++ 4 files changed, 179 insertions(+) diff --git a/payments/db/sql_store.go b/payments/db/sql_store.go index 3637a981b..7f8f9e666 100644 --- a/payments/db/sql_store.go +++ b/payments/db/sql_store.go @@ -87,6 +87,25 @@ type SQLQueries interface { // DeleteFailedAttempts removes all failed HTLCs from the db for a // given payment. DeleteFailedAttempts(ctx context.Context, paymentID int64) error + + /* + Migration specific queries. + + These queries are used ONLY for the one-time migration from KV + to SQL. + */ + + // FetchPaymentsByIDsMig is a migration-only batch fetch that returns + // payment data along with HTLC attempt counts for structural + // validation. + FetchPaymentsByIDsMig(ctx context.Context, paymentIDs []int64) ([]sqlc.FetchPaymentsByIDsMigRow, error) + + // InsertPaymentMig is a migration-only variant of InsertPayment that + // allows setting fail_reason when inserting historical payments, since + // for real payments they have not failed at creation time and so no + // failure reason would exist yet. + InsertPaymentMig(ctx context.Context, arg sqlc.InsertPaymentMigParams) (int64, error) + } // BatchedSQLQueries is a version of the SQLQueries that's capable diff --git a/sqldb/sqlc/payments.sql.go b/sqldb/sqlc/payments.sql.go index b9ec31499..fe02be487 100644 --- a/sqldb/sqlc/payments.sql.go +++ b/sqldb/sqlc/payments.sql.go @@ -597,6 +597,72 @@ func (q *Queries) FetchPaymentsByIDs(ctx context.Context, paymentIds []int64) ([ return items, nil } +const fetchPaymentsByIDsMig = `-- name: FetchPaymentsByIDsMig :many +SELECT + p.id, + p.amount_msat, + p.created_at, + p.payment_identifier, + p.fail_reason, + COUNT(ha.id) AS htlc_attempt_count +FROM payments p +LEFT JOIN payment_htlc_attempts ha ON ha.payment_id = p.id +WHERE p.id IN (/*SLICE:payment_ids*/?) +GROUP BY p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason +ORDER BY p.id ASC +` + +type FetchPaymentsByIDsMigRow struct { + ID int64 + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 + HtlcAttemptCount int64 +} + +// Migration-specific batch fetch that returns payment data along with HTLC +// attempt counts for structural validation during KV to SQL migration. +func (q *Queries) FetchPaymentsByIDsMig(ctx context.Context, paymentIds []int64) ([]FetchPaymentsByIDsMigRow, error) { + query := fetchPaymentsByIDsMig + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchPaymentsByIDsMigRow + for rows.Next() { + var i FetchPaymentsByIDsMigRow + if err := rows.Scan( + &i.ID, + &i.AmountMsat, + &i.CreatedAt, + &i.PaymentIdentifier, + &i.FailReason, + &i.HtlcAttemptCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const fetchRouteLevelFirstHopCustomRecords = `-- name: FetchRouteLevelFirstHopCustomRecords :many SELECT l.id, @@ -918,6 +984,51 @@ func (q *Queries) InsertPaymentIntent(ctx context.Context, arg InsertPaymentInte return id, err } +const insertPaymentMig = `-- name: InsertPaymentMig :one +/* ───────────────────────────────────────────── + Migration-specific queries + + These queries are used ONLY for the one-time migration from KV to SQL. + ───────────────────────────────────────────── +*/ + +INSERT INTO payments ( + amount_msat, + created_at, + payment_identifier, + fail_reason) +VALUES ( + $1, + $2, + $3, + $4 +) +RETURNING id +` + +type InsertPaymentMigParams struct { + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 +} + +// Migration-specific payment insert that allows setting fail_reason. +// Normal InsertPayment forces fail_reason to NULL since new payments +// aren't failed yet. During migration, we're inserting historical data +// that may already be failed. +func (q *Queries) InsertPaymentMig(ctx context.Context, arg InsertPaymentMigParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPaymentMig, + arg.AmountMsat, + arg.CreatedAt, + arg.PaymentIdentifier, + arg.FailReason, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + const insertRouteHop = `-- name: InsertRouteHop :one INSERT INTO payment_route_hops ( htlc_attempt_index, diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 3be738ba9..02f22971c 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -54,6 +54,9 @@ type Querier interface { // Used to avoid fetching redundant payment data when processing multiple // attempts for the same payment. FetchPaymentsByIDs(ctx context.Context, paymentIds []int64) ([]FetchPaymentsByIDsRow, error) + // Migration-specific batch fetch that returns payment data along with HTLC + // attempt counts for structural validation during KV to SQL migration. + FetchPaymentsByIDsMig(ctx context.Context, paymentIds []int64) ([]FetchPaymentsByIDsMigRow, error) // FetchPendingInvoices returns all invoices in a pending state (open or // accepted). The invoices_state_idx index on the state column makes this a // fast index scan rather than a full table scan. @@ -178,6 +181,11 @@ type Querier interface { InsertPaymentHopCustomRecord(ctx context.Context, arg InsertPaymentHopCustomRecordParams) error // Insert a payment intent for a given payment and return its ID. InsertPaymentIntent(ctx context.Context, arg InsertPaymentIntentParams) (int64, error) + // Migration-specific payment insert that allows setting fail_reason. + // Normal InsertPayment forces fail_reason to NULL since new payments + // aren't failed yet. During migration, we're inserting historical data + // that may already be failed. + InsertPaymentMig(ctx context.Context, arg InsertPaymentMigParams) (int64, error) InsertRouteHop(ctx context.Context, arg InsertRouteHopParams) (int64, error) InsertRouteHopAmp(ctx context.Context, arg InsertRouteHopAmpParams) error InsertRouteHopBlinded(ctx context.Context, arg InsertRouteHopBlindedParams) error diff --git a/sqldb/sqlc/queries/payments.sql b/sqldb/sqlc/queries/payments.sql index 419f7bf1a..46f84e494 100644 --- a/sqldb/sqlc/queries/payments.sql +++ b/sqldb/sqlc/queries/payments.sql @@ -368,3 +368,44 @@ VALUES ( -- name: FailPayment :execresult UPDATE payments SET fail_reason = $1 WHERE payment_identifier = $2; + +/* ───────────────────────────────────────────── + Migration-specific queries + + These queries are used ONLY for the one-time migration from KV to SQL. + ───────────────────────────────────────────── +*/ + +-- name: InsertPaymentMig :one +-- Migration-specific payment insert that allows setting fail_reason. +-- Normal InsertPayment forces fail_reason to NULL since new payments +-- aren't failed yet. During migration, we're inserting historical data +-- that may already be failed. +INSERT INTO payments ( + amount_msat, + created_at, + payment_identifier, + fail_reason) +VALUES ( + @amount_msat, + @created_at, + @payment_identifier, + @fail_reason +) +RETURNING id; + +-- name: FetchPaymentsByIDsMig :many +-- Migration-specific batch fetch that returns payment data along with HTLC +-- attempt counts for structural validation during KV to SQL migration. +SELECT + p.id, + p.amount_msat, + p.created_at, + p.payment_identifier, + p.fail_reason, + COUNT(ha.id) AS htlc_attempt_count +FROM payments p +LEFT JOIN payment_htlc_attempts ha ON ha.payment_id = p.id +WHERE p.id IN (sqlc.slice('payment_ids')/*SLICE:payment_ids*/) +GROUP BY p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason +ORDER BY p.id ASC; \ No newline at end of file From 74f8f2d9e6f545fa7b065b6d4d9f14056f027c89 Mon Sep 17 00:00:00 2001 From: ziggie Date: Sat, 10 Jan 2026 23:46:17 +0100 Subject: [PATCH 02/16] sqldb+payments: add payment_duplicates for legacy duplicate payments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Older LND versions could create multiple payments for the same hash. We need to preserve those historical records during KV→SQL migration, but they don’t fit the normal payment schema because we enforce a unique payment hash constraint. Introduce a lean payment_duplicates table to store only the essential fields (identifier, amount, timestamps, settle/fail data). This keeps the primary payment records stable and makes the migration deterministic even when duplicate records lack attempt info. The table is intentionally minimal and can be dropped after migration if no duplicate payments exist. For now there is no logic in place which allows the noderunner to fetch duplicate payments after the migration. --- payments/db/sql_store.go | 5 + .../000011_payment_duplicates.down.sql | 2 + .../000011_payment_duplicates.up.sql | 43 +++++++++ sqldb/sqlc/models.go | 10 ++ sqldb/sqlc/payments.sql.go | 92 +++++++++++++++++++ sqldb/sqlc/querier.go | 6 ++ sqldb/sqlc/queries/payments.sql | 38 +++++++- 7 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 sqldb/sqlc/migrations/000011_payment_duplicates.down.sql create mode 100644 sqldb/sqlc/migrations/000011_payment_duplicates.up.sql diff --git a/payments/db/sql_store.go b/payments/db/sql_store.go index 7f8f9e666..2c8dde93d 100644 --- a/payments/db/sql_store.go +++ b/payments/db/sql_store.go @@ -57,6 +57,8 @@ type SQLQueries interface { FetchAllInflightAttempts(ctx context.Context, arg sqlc.FetchAllInflightAttemptsParams) ([]sqlc.PaymentHtlcAttempt, error) FetchHopsForAttempts(ctx context.Context, htlcAttemptIndices []int64) ([]sqlc.FetchHopsForAttemptsRow, error) + FetchPaymentDuplicates(ctx context.Context, paymentID int64) ([]sqlc.PaymentDuplicate, error) + FetchPaymentLevelFirstHopCustomRecords(ctx context.Context, paymentIDs []int64) ([]sqlc.PaymentFirstHopCustomRecord, error) FetchRouteLevelFirstHopCustomRecords(ctx context.Context, htlcAttemptIndices []int64) ([]sqlc.PaymentAttemptFirstHopCustomRecord, error) FetchHopLevelCustomRecords(ctx context.Context, hopIDs []int64) ([]sqlc.PaymentHopCustomRecord, error) @@ -106,6 +108,9 @@ type SQLQueries interface { // failure reason would exist yet. InsertPaymentMig(ctx context.Context, arg sqlc.InsertPaymentMigParams) (int64, error) + // InsertPaymentDuplicateMig inserts a duplicate payment record during + // migration. + InsertPaymentDuplicateMig(ctx context.Context, arg sqlc.InsertPaymentDuplicateMigParams) (int64, error) } // BatchedSQLQueries is a version of the SQLQueries that's capable diff --git a/sqldb/sqlc/migrations/000011_payment_duplicates.down.sql b/sqldb/sqlc/migrations/000011_payment_duplicates.down.sql new file mode 100644 index 000000000..39c4a313a --- /dev/null +++ b/sqldb/sqlc/migrations/000011_payment_duplicates.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_payment_duplicates_payment_id; +DROP TABLE IF EXISTS payment_duplicates; diff --git a/sqldb/sqlc/migrations/000011_payment_duplicates.up.sql b/sqldb/sqlc/migrations/000011_payment_duplicates.up.sql new file mode 100644 index 000000000..8abaf3e09 --- /dev/null +++ b/sqldb/sqlc/migrations/000011_payment_duplicates.up.sql @@ -0,0 +1,43 @@ +-- ───────────────────────────────────────────── +-- Payment Duplicate Records Table +-- ───────────────────────────────────────────── +-- Stores duplicate payment records that were created in older versions +-- of lnd. This table is intentionally minimal and is expected to be dropped +-- in the future especially if no duplicates were migrated. +-- ───────────────────────────────────────────── + +CREATE TABLE IF NOT EXISTS payment_duplicates ( + -- Primary key for the duplicate record. + id INTEGER PRIMARY KEY, + + -- Reference to the primary payment this duplicate belongs to. + payment_id BIGINT NOT NULL REFERENCES payments (id) ON DELETE CASCADE, + + -- Amount of the duplicate payment in millisatoshis. + amount_msat BIGINT NOT NULL, + + -- Timestamp when the duplicate payment was created. + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + + -- Failure reason for failed payments (if known). + fail_reason INTEGER, + + -- Settlement payload for succeeded payments (if known). + settle_preimage BLOB, + + -- Settlement time for succeeded payments (if known). + settle_time TIMESTAMP, + + -- Ensure we record either a failure reason or settlement data. + -- During the migration if we encounter a duplicate payment that has no + -- failure reason or settlement data, we will mark it as failed. Duplicate + -- payments were a bug in older versions of LND, so we can be sure if a + -- duplicate payment has no failure reason or settlement data, the + -- corresponding HTLC has been failed. + CONSTRAINT chk_payment_duplicates_outcome + CHECK (fail_reason IS NOT NULL OR settle_preimage IS NOT NULL) +); + +-- Index for efficient lookup by primary payment. +CREATE INDEX IF NOT EXISTS idx_payment_duplicates_payment_id +ON payment_duplicates(payment_id); diff --git a/sqldb/sqlc/models.go b/sqldb/sqlc/models.go index 0e04cd467..513f045ba 100644 --- a/sqldb/sqlc/models.go +++ b/sqldb/sqlc/models.go @@ -224,6 +224,16 @@ type PaymentAttemptFirstHopCustomRecord struct { Value []byte } +type PaymentDuplicate struct { + ID int64 + PaymentID int64 + AmountMsat int64 + CreatedAt time.Time + FailReason sql.NullInt32 + SettlePreimage []byte + SettleTime sql.NullTime +} + type PaymentFirstHopCustomRecord struct { ID int64 PaymentID int64 diff --git a/sqldb/sqlc/payments.sql.go b/sqldb/sqlc/payments.sql.go index fe02be487..5ac3d826a 100644 --- a/sqldb/sqlc/payments.sql.go +++ b/sqldb/sqlc/payments.sql.go @@ -479,6 +479,53 @@ func (q *Queries) FetchPayment(ctx context.Context, paymentIdentifier []byte) (F return i, err } +const fetchPaymentDuplicates = `-- name: FetchPaymentDuplicates :many +SELECT + id, + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +FROM payment_duplicates +WHERE payment_id = $1 +ORDER BY id ASC +` + +// Fetch all duplicate payment records from the payment_duplicates table for +// a given payment ID. +func (q *Queries) FetchPaymentDuplicates(ctx context.Context, paymentID int64) ([]PaymentDuplicate, error) { + rows, err := q.db.QueryContext(ctx, fetchPaymentDuplicates, paymentID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentDuplicate + for rows.Next() { + var i PaymentDuplicate + if err := rows.Scan( + &i.ID, + &i.PaymentID, + &i.AmountMsat, + &i.CreatedAt, + &i.FailReason, + &i.SettlePreimage, + &i.SettleTime, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const fetchPaymentLevelFirstHopCustomRecords = `-- name: FetchPaymentLevelFirstHopCustomRecords :many SELECT l.id, @@ -909,6 +956,51 @@ func (q *Queries) InsertPaymentAttemptFirstHopCustomRecord(ctx context.Context, return err } +const insertPaymentDuplicateMig = `-- name: InsertPaymentDuplicateMig :one +INSERT INTO payment_duplicates ( + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6 +) +RETURNING id +` + +type InsertPaymentDuplicateMigParams struct { + PaymentID int64 + AmountMsat int64 + CreatedAt time.Time + FailReason sql.NullInt32 + SettlePreimage []byte + SettleTime sql.NullTime +} + +// Insert a duplicate payment record into the payment_duplicates table and +// return its ID. +func (q *Queries) InsertPaymentDuplicateMig(ctx context.Context, arg InsertPaymentDuplicateMigParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPaymentDuplicateMig, + arg.PaymentID, + arg.AmountMsat, + arg.CreatedAt, + arg.FailReason, + arg.SettlePreimage, + arg.SettleTime, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + const insertPaymentFirstHopCustomRecord = `-- name: InsertPaymentFirstHopCustomRecord :exec INSERT INTO payment_first_hop_custom_records ( payment_id, diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 02f22971c..947237300 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -49,6 +49,9 @@ type Querier interface { FetchHtlcAttemptResolutionsForPayments(ctx context.Context, paymentIds []int64) ([]FetchHtlcAttemptResolutionsForPaymentsRow, error) FetchHtlcAttemptsForPayments(ctx context.Context, paymentIds []int64) ([]FetchHtlcAttemptsForPaymentsRow, error) FetchPayment(ctx context.Context, paymentIdentifier []byte) (FetchPaymentRow, error) + // Fetch all duplicate payment records from the payment_duplicates table for + // a given payment ID. + FetchPaymentDuplicates(ctx context.Context, paymentID int64) ([]PaymentDuplicate, error) FetchPaymentLevelFirstHopCustomRecords(ctx context.Context, paymentIds []int64) ([]PaymentFirstHopCustomRecord, error) // Batch fetch payment and intent data for a set of payment IDs. // Used to avoid fetching redundant payment data when processing multiple @@ -177,6 +180,9 @@ type Querier interface { // payment process. InsertPayment(ctx context.Context, arg InsertPaymentParams) (int64, error) InsertPaymentAttemptFirstHopCustomRecord(ctx context.Context, arg InsertPaymentAttemptFirstHopCustomRecordParams) error + // Insert a duplicate payment record into the payment_duplicates table and + // return its ID. + InsertPaymentDuplicateMig(ctx context.Context, arg InsertPaymentDuplicateMigParams) (int64, error) InsertPaymentFirstHopCustomRecord(ctx context.Context, arg InsertPaymentFirstHopCustomRecordParams) error InsertPaymentHopCustomRecord(ctx context.Context, arg InsertPaymentHopCustomRecordParams) error // Insert a payment intent for a given payment and return its ID. diff --git a/sqldb/sqlc/queries/payments.sql b/sqldb/sqlc/queries/payments.sql index 46f84e494..1bb2c50ff 100644 --- a/sqldb/sqlc/queries/payments.sql +++ b/sqldb/sqlc/queries/payments.sql @@ -40,6 +40,21 @@ FROM payments p LEFT JOIN payment_intents i ON i.payment_id = p.id WHERE p.payment_identifier = $1; +-- name: FetchPaymentDuplicates :many +-- Fetch all duplicate payment records from the payment_duplicates table for +-- a given payment ID. +SELECT + id, + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +FROM payment_duplicates +WHERE payment_id = $1 +ORDER BY id ASC; + -- name: CountPayments :one SELECT COUNT(*) FROM payments; @@ -408,4 +423,25 @@ FROM payments p LEFT JOIN payment_htlc_attempts ha ON ha.payment_id = p.id WHERE p.id IN (sqlc.slice('payment_ids')/*SLICE:payment_ids*/) GROUP BY p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason -ORDER BY p.id ASC; \ No newline at end of file +ORDER BY p.id ASC; + +-- name: InsertPaymentDuplicateMig :one +-- Insert a duplicate payment record into the payment_duplicates table and +-- return its ID. +INSERT INTO payment_duplicates ( + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +) +VALUES ( + @payment_id, + @amount_msat, + @created_at, + @fail_reason, + @settle_preimage, + @settle_time +) +RETURNING id; From fb705bb0f925beea2ab8f03d85bf44a4c9b11f9c Mon Sep 17 00:00:00 2001 From: ziggie Date: Sat, 10 Jan 2026 23:58:49 +0100 Subject: [PATCH 03/16] payments/migration1: freeze core payment code Copy the core payments/db code into payments/db/migration1 and add the required sqlc-generated types/queries from sqldb/sqlc. This effectively freezes the migration code so it stays robust against future query or schema changes in the main payments package. Replace the delegation to channeldb.ReadElement/WriteElement with self-contained, frozen implementations that only handle the exact types required by this migration package. This removes the dependency on the live channeldb codec so that future changes to channeldb serialization cannot silently corrupt or break the migration. UnknownElementType is also defined locally for the same reason. --- payments/db/migration1/codec.go | 198 ++ payments/db/migration1/errors.go | 154 ++ payments/db/migration1/interface.go | 140 ++ .../db/migration1/kv_duplicate_payments.go | 249 ++ payments/db/migration1/kv_store.go | 2120 +++++++++++++++++ payments/db/migration1/log.go | 32 + payments/db/migration1/options.go | 26 + payments/db/migration1/payment.go | 689 ++++++ payments/db/migration1/payment_status.go | 257 ++ payments/db/migration1/query.go | 75 + payments/db/migration1/route.go | 138 ++ payments/db/migration1/sql_converters.go | 274 +++ payments/db/migration1/sql_store.go | 2019 ++++++++++++++++ payments/db/migration1/sqlc/db.go | 31 + payments/db/migration1/sqlc/db_custom.go | 123 + payments/db/migration1/sqlc/models.go | 110 + payments/db/migration1/sqlc/payments.sql.go | 1357 +++++++++++ 17 files changed, 7992 insertions(+) create mode 100644 payments/db/migration1/codec.go create mode 100644 payments/db/migration1/errors.go create mode 100644 payments/db/migration1/interface.go create mode 100644 payments/db/migration1/kv_duplicate_payments.go create mode 100644 payments/db/migration1/kv_store.go create mode 100644 payments/db/migration1/log.go create mode 100644 payments/db/migration1/options.go create mode 100644 payments/db/migration1/payment.go create mode 100644 payments/db/migration1/payment_status.go create mode 100644 payments/db/migration1/query.go create mode 100644 payments/db/migration1/route.go create mode 100644 payments/db/migration1/sql_converters.go create mode 100644 payments/db/migration1/sql_store.go create mode 100644 payments/db/migration1/sqlc/db.go create mode 100644 payments/db/migration1/sqlc/db_custom.go create mode 100644 payments/db/migration1/sqlc/models.go create mode 100644 payments/db/migration1/sqlc/payments.sql.go diff --git a/payments/db/migration1/codec.go b/payments/db/migration1/codec.go new file mode 100644 index 000000000..acc173bc4 --- /dev/null +++ b/payments/db/migration1/codec.go @@ -0,0 +1,198 @@ +package migration1 + +import ( + "encoding/binary" + "fmt" + "io" + "time" + + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/lnwire" +) + +// Big endian is the preferred byte order, due to cursor scans over +// integer keys iterating in order. +var byteOrder = binary.BigEndian + +// UnknownElementType is an error returned when the codec is unable to encode +// or decode a particular type. +// +// NOTE: This is a frozen, self-contained copy of the serialization error type +// to ensure that this migration package has no dependency on live packages +// whose implementation may change after this migration was released. +type UnknownElementType struct { + method string + element interface{} +} + +// NewUnknownElementType creates a new UnknownElementType error from the passed +// method name and element. +func NewUnknownElementType(method string, el interface{}) UnknownElementType { + return UnknownElementType{method: method, element: el} +} + +// Error returns the name of the method that encountered the error, as well as +// the type that was unsupported. +func (e UnknownElementType) Error() string { + return fmt.Sprintf("Unknown type in %s: %T", e.method, e.element) +} + +// WriteElement serializes a single element into the provided io.Writer using +// big-endian byte order. Only the types required by the migration1 package are +// supported. This is a frozen, self-contained copy of the serialization logic +// to ensure that this migration package has no dependency on live packages +// whose implementation may change after this migration was released. +func WriteElement(w io.Writer, element interface{}) error { + switch e := element.(type) { + case paymentIndexType: + if err := binary.Write(w, byteOrder, e); err != nil { + return err + } + + case uint8: + if err := binary.Write(w, byteOrder, e); err != nil { + return err + } + + case uint32: + if err := binary.Write(w, byteOrder, e); err != nil { + return err + } + + case uint64: + if err := binary.Write(w, byteOrder, e); err != nil { + return err + } + + case lnwire.MilliSatoshi: + if err := binary.Write(w, byteOrder, uint64(e)); err != nil { + return err + } + + case [32]byte: + if _, err := w.Write(e[:]); err != nil { + return err + } + + case []byte: + if err := wire.WriteVarBytes(w, 0, e); err != nil { + return err + } + + default: + return UnknownElementType{"WriteElement", e} + } + + return nil +} + +// WriteElements serializes a variadic list of elements into the given +// io.Writer. +func WriteElements(w io.Writer, elements ...interface{}) error { + for _, element := range elements { + if err := WriteElement(w, element); err != nil { + return err + } + } + + return nil +} + +// ReadElement deserializes a single element from the provided io.Reader using +// big-endian byte order. Only the types required by the migration1 package are +// supported. This is a frozen, self-contained copy of the deserialization +// logic to ensure that this migration package has no dependency on live +// packages whose implementation may change after this migration was released. +func ReadElement(r io.Reader, element interface{}) error { + switch e := element.(type) { + case *paymentIndexType: + if err := binary.Read(r, byteOrder, e); err != nil { + return err + } + + case *uint8: + if err := binary.Read(r, byteOrder, e); err != nil { + return err + } + + case *uint32: + if err := binary.Read(r, byteOrder, e); err != nil { + return err + } + + case *uint64: + if err := binary.Read(r, byteOrder, e); err != nil { + return err + } + + case *lnwire.MilliSatoshi: + var a uint64 + if err := binary.Read(r, byteOrder, &a); err != nil { + return err + } + *e = lnwire.MilliSatoshi(a) + + case *[32]byte: + if _, err := io.ReadFull(r, e[:]); err != nil { + return err + } + + case *[]byte: + b, err := wire.ReadVarBytes(r, 0, 66000, "[]byte") + if err != nil { + return err + } + *e = b + + default: + return UnknownElementType{"ReadElement", e} + } + + return nil +} + +// ReadElements deserializes the provided io.Reader into a variadic list of +// target elements. +func ReadElements(r io.Reader, elements ...interface{}) error { + for _, element := range elements { + if err := ReadElement(r, element); err != nil { + return err + } + } + + return nil +} + +// deserializeTime deserializes time as unix nanoseconds. +func deserializeTime(r io.Reader) (time.Time, error) { + var scratch [8]byte + if _, err := io.ReadFull(r, scratch[:]); err != nil { + return time.Time{}, err + } + + // Convert to time.Time. Interpret unix nano time zero as a zero + // time.Time value. + unixNano := byteOrder.Uint64(scratch[:]) + if unixNano == 0 { + return time.Time{}, nil + } + + return time.Unix(0, int64(unixNano)), nil +} + +// serializeTime serializes time as unix nanoseconds. +func serializeTime(w io.Writer, t time.Time) error { + var scratch [8]byte + + // Convert to unix nano seconds, but only if time is non-zero. Calling + // UnixNano() on a zero time yields an undefined result. + var unixNano int64 + if !t.IsZero() { + unixNano = t.UnixNano() + } + + byteOrder.PutUint64(scratch[:], uint64(unixNano)) + _, err := w.Write(scratch[:]) + + return err +} diff --git a/payments/db/migration1/errors.go b/payments/db/migration1/errors.go new file mode 100644 index 000000000..7c2c98288 --- /dev/null +++ b/payments/db/migration1/errors.go @@ -0,0 +1,154 @@ +package migration1 + +import "errors" + +var ( + // ErrAlreadyPaid signals we have already paid this payment hash. + ErrAlreadyPaid = errors.New("invoice is already paid") + + // ErrPaymentInFlight signals that payment for this payment hash is + // already "in flight" on the network. + ErrPaymentInFlight = errors.New("payment is in transition") + + // ErrPaymentExists is returned when we try to initialize an already + // existing payment that is not failed. + ErrPaymentExists = errors.New("payment already exists") + + // ErrPaymentInternal is returned when performing the payment has a + // conflicting state, such as, + // - payment has StatusSucceeded but remaining amount is not zero. + // - payment has StatusInitiated but remaining amount is zero. + // - payment has StatusFailed but remaining amount is zero. + ErrPaymentInternal = errors.New("internal error") + + // ErrPaymentNotInitiated is returned if the payment wasn't initiated. + ErrPaymentNotInitiated = errors.New("payment isn't initiated") + + // ErrPaymentAlreadySucceeded is returned in the event we attempt to + // change the status of a payment already succeeded. + ErrPaymentAlreadySucceeded = errors.New("payment is already succeeded") + + // ErrPaymentAlreadyFailed is returned in the event we attempt to alter + // a failed payment. + ErrPaymentAlreadyFailed = errors.New("payment has already failed") + + // ErrUnknownPaymentStatus is returned when we do not recognize the + // existing state of a payment. + ErrUnknownPaymentStatus = errors.New("unknown payment status") + + // ErrPaymentTerminal is returned if we attempt to alter a payment that + // already has reached a terminal condition. + ErrPaymentTerminal = errors.New("payment has reached terminal " + + "condition") + + // ErrAttemptAlreadySettled is returned if we try to alter an already + // settled HTLC attempt. + ErrAttemptAlreadySettled = errors.New("attempt already settled") + + // ErrAttemptAlreadyFailed is returned if we try to alter an already + // failed HTLC attempt. + ErrAttemptAlreadyFailed = errors.New("attempt already failed") + + // ErrValueMismatch is returned if we try to register a non-MPP attempt + // with an amount that doesn't match the payment amount. + ErrValueMismatch = errors.New("attempted value doesn't match payment " + + "amount") + + // ErrValueExceedsAmt is returned if we try to register an attempt that + // would take the total sent amount above the payment amount. + ErrValueExceedsAmt = errors.New("attempted value exceeds payment " + + "amount") + + // ErrNonMPPayment is returned if we try to register an MPP attempt for + // a payment that already has a non-MPP attempt registered. + ErrNonMPPayment = errors.New("payment has non-MPP attempts") + + // ErrMPPayment is returned if we try to register a non-MPP attempt for + // a payment that already has an MPP attempt registered. + ErrMPPayment = errors.New("payment has MPP attempts") + + // ErrMPPRecordInBlindedPayment is returned if we try to register an + // attempt with an MPP record for a payment to a blinded path. + ErrMPPRecordInBlindedPayment = errors.New("blinded payment cannot " + + "contain MPP records") + + // ErrBlindedPaymentTotalAmountMismatch is returned if we try to + // register an HTLC shard to a blinded route where the total amount + // doesn't match existing shards. + ErrBlindedPaymentTotalAmountMismatch = errors.New("blinded path " + + "total amount mismatch") + + // ErrMixedBlindedAndNonBlindedPayments is returned if we try to + // register a non-blinded attempt to a payment which uses a blinded + // paths or vice versa. + ErrMixedBlindedAndNonBlindedPayments = errors.New("mixed blinded and " + + "non-blinded payments") + + // ErrBlindedPaymentMissingTotalAmount is returned if we try to + // register a blinded payment attempt where the final hop doesn't set + // the total amount. + ErrBlindedPaymentMissingTotalAmount = errors.New("blinded payment " + + "final hop must set total amount") + + // ErrMPPPaymentAddrMismatch is returned if we try to register an MPP + // shard where the payment address doesn't match existing shards. + ErrMPPPaymentAddrMismatch = errors.New("payment address mismatch") + + // ErrMPPTotalAmountMismatch is returned if we try to register an MPP + // shard where the total amount doesn't match existing shards. + ErrMPPTotalAmountMismatch = errors.New("mp payment total amount " + + "mismatch") + + // ErrPaymentPendingSettled is returned when we try to add a new + // attempt to a payment that has at least one of its HTLCs settled. + ErrPaymentPendingSettled = errors.New("payment has settled htlcs") + + // ErrPaymentPendingFailed is returned when we try to add a new attempt + // to a payment that already has a failure reason. + ErrPaymentPendingFailed = errors.New("payment has failure reason") + + // ErrSentExceedsTotal is returned if the payment's current total sent + // amount exceed the total amount. + ErrSentExceedsTotal = errors.New("total sent exceeds total amount") + + // ErrNoAttemptInfo is returned when no attempt info is stored yet. + ErrNoAttemptInfo = errors.New("unable to find attempt info for " + + "inflight payment") +) + +// KV backend specific errors. +var ( + // ErrNoSequenceNumber is returned if we look up a payment which does + // not have a sequence number. + ErrNoSequenceNumber = errors.New("sequence number not found") + + // ErrDuplicateNotFound is returned when we lookup a payment by its + // index and cannot find a payment with a matching sequence number. + ErrDuplicateNotFound = errors.New("duplicate payment not found") + + // ErrNoDuplicateBucket is returned when we expect to find duplicates + // when looking up a payment from its index, but the payment does not + // have any. + ErrNoDuplicateBucket = errors.New("expected duplicate bucket") + + // ErrNoDuplicateNestedBucket is returned if we do not find duplicate + // payments in their own sub-bucket. + ErrNoDuplicateNestedBucket = errors.New("nested duplicate bucket not " + + "found") + + // ErrNoDuplicateSequenceNumber is returned when a duplicate payment + // sub-bucket does not contain the sequence number key. + ErrNoDuplicateSequenceNumber = errors.New("duplicate payment " + + "sequence number not found") + + // ErrNoSequenceNrIndex is returned when an attempt to lookup a payment + // index is made for a sequence number that is not indexed. + // + // NOTE: Only used for the kv backend. + ErrNoSequenceNrIndex = errors.New("payment sequence number index " + + "does not exist") + + // errMaxPaymentsReached is used internally to signal that the maximum + // number of payments has been reached during a paginated query. + errMaxPaymentsReached = errors.New("max payments reached") +) diff --git a/payments/db/migration1/interface.go b/payments/db/migration1/interface.go new file mode 100644 index 000000000..7d47118e9 --- /dev/null +++ b/payments/db/migration1/interface.go @@ -0,0 +1,140 @@ +package migration1 + +import ( + "context" + + "github.com/lightningnetwork/lnd/lntypes" +) + +// DB represents the interface to the underlying payments database. +type DB interface { + PaymentReader + PaymentWriter +} + +// PaymentReader represents the interface to read operations from the payments +// database. +type PaymentReader interface { + // QueryPayments queries the payments database and should support + // pagination. + QueryPayments(ctx context.Context, query Query) (Response, error) + + // FetchPayment fetches the payment corresponding to the given payment + // hash. + FetchPayment(ctx context.Context, + paymentHash lntypes.Hash) (*MPPayment, error) + + // FetchInFlightPayments returns all payments with status InFlight. + FetchInFlightPayments(ctx context.Context) ([]*MPPayment, error) +} + +// PaymentWriter represents the interface to write operations to the payments +// database. +type PaymentWriter interface { + // DeletePayment deletes a payment from the DB given its payment hash. + DeletePayment(ctx context.Context, paymentHash lntypes.Hash, + failedAttemptsOnly bool) error + + // DeletePayments deletes all payments from the DB given the specified + // flags. + DeletePayments(ctx context.Context, failedOnly, + failedAttemptsOnly bool) (int, error) + + PaymentControl +} + +// PaymentControl represents the interface to control the payment lifecycle and +// its database operations. This interface represents the control flow of how +// a payment should be handled in the database. They are not just writing +// operations but they inherently represent the flow of a payment. The methods +// are called in the following order. +// +// 1. InitPayment. +// 2. RegisterAttempt (a payment can have multiple attempts). +// 3. SettleAttempt or FailAttempt (attempts can also fail as long as the +// sending amount will be eventually settled). +// 4. Payment succeeds or "Fail" is called. +// 5. DeleteFailedAttempts is called which will delete all failed attempts +// for a payment to clean up the database. +type PaymentControl interface { + // InitPayment checks that no other payment with the same payment hash + // exists in the database before creating a new payment. However, it + // should allow the user making a subsequent payment if the payment is + // in a Failed state. + InitPayment(context.Context, lntypes.Hash, *PaymentCreationInfo) error + + // RegisterAttempt atomically records the provided HTLCAttemptInfo. + // + // IMPORTANT: Callers MUST serialize calls to RegisterAttempt for the + // same payment hash. Concurrent calls will result in race conditions + // where both calls read the same initial payment state, validate + // against stale data, and could cause overpayment. For example: + // - Both goroutines fetch payment with 400 sats sent + // - Both validate sending 650 sats won't overpay (within limit) + // - Both commit successfully + // - Result: 1700 sats sent, exceeding the payment amount + // The payment router/controller layer is responsible for ensuring + // serialized access per payment hash. + RegisterAttempt(context.Context, lntypes.Hash, + *HTLCAttemptInfo) (*MPPayment, error) + + // SettleAttempt marks the given attempt settled with the preimage. If + // this is a multi shard payment, this might implicitly mean the + // full payment succeeded. + // + // After invoking this method, InitPayment should always return an + // error to prevent us from making duplicate payments to the same + // payment hash. The provided preimage is atomically saved to the DB + // for record keeping. + SettleAttempt(context.Context, lntypes.Hash, uint64, + *HTLCSettleInfo) (*MPPayment, error) + + // FailAttempt marks the given payment attempt failed. + FailAttempt(context.Context, lntypes.Hash, uint64, + *HTLCFailInfo) (*MPPayment, error) + + // Fail transitions a payment into the Failed state, and records + // the ultimate reason the payment failed. Note that this should only + // be called when all active attempts are already failed. After + // invoking this method, InitPayment should return nil on its next call + // for this payment hash, allowing the user to make a subsequent + // payment. + Fail(context.Context, lntypes.Hash, FailureReason) (*MPPayment, error) + + // DeleteFailedAttempts removes all failed HTLCs from the db. It should + // be called for a given payment whenever all inflight htlcs are + // completed, and the payment has reached a final terminal state. + DeleteFailedAttempts(context.Context, lntypes.Hash) error +} + +// DBMPPayment is an interface that represents the payment state during a +// payment lifecycle. +type DBMPPayment interface { + // GetState returns the current state of the payment. + GetState() *MPPaymentState + + // Terminated returns true if the payment is in a final state. + Terminated() bool + + // GetStatus returns the current status of the payment. + GetStatus() PaymentStatus + + // NeedWaitAttempts specifies whether the payment needs to wait for the + // outcome of an attempt. + NeedWaitAttempts() (bool, error) + + // GetHTLCs returns all HTLCs of this payment. + GetHTLCs() []HTLCAttempt + + // InFlightHTLCs returns all HTLCs that are in flight. + InFlightHTLCs() []HTLCAttempt + + // AllowMoreAttempts is used to decide whether we can safely attempt + // more HTLCs for a given payment state. Return an error if the payment + // is in an unexpected state. + AllowMoreAttempts() (bool, error) + + // TerminalInfo returns the settled HTLC attempt or the payment's + // failure reason. + TerminalInfo() (*HTLCAttempt, *FailureReason) +} diff --git a/payments/db/migration1/kv_duplicate_payments.go b/payments/db/migration1/kv_duplicate_payments.go new file mode 100644 index 000000000..f72848da4 --- /dev/null +++ b/payments/db/migration1/kv_duplicate_payments.go @@ -0,0 +1,249 @@ +package migration1 + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" +) + +var ( + // duplicatePaymentsBucket is the name of a optional sub-bucket within + // the payment hash bucket, that is used to hold duplicate payments to a + // payment hash. This is needed to support information from earlier + // versions of lnd, where it was possible to pay to a payment hash more + // than once. + duplicatePaymentsBucket = []byte("payment-duplicate-bucket") + + // duplicatePaymentSettleInfoKey is a key used in the payment's + // sub-bucket to store the settle info of the payment. + duplicatePaymentSettleInfoKey = []byte("payment-settle-info") + + // duplicatePaymentAttemptInfoKey is a key used in the payment's + // sub-bucket to store the info about the latest attempt that was done + // for the payment in question. + duplicatePaymentAttemptInfoKey = []byte("payment-attempt-info") + + // duplicatePaymentCreationInfoKey is a key used in the payment's + // sub-bucket to store the creation info of the payment. + duplicatePaymentCreationInfoKey = []byte("payment-creation-info") + + // duplicatePaymentFailInfoKey is a key used in the payment's sub-bucket + // to store information about the reason a payment failed. + duplicatePaymentFailInfoKey = []byte("payment-fail-info") + + // duplicatePaymentSequenceKey is a key used in the payment's sub-bucket + // to store the sequence number of the payment. + duplicatePaymentSequenceKey = []byte("payment-sequence-key") +) + +// duplicateHTLCAttemptInfo contains static information about a specific HTLC +// attempt for a payment. This information is used by the router to handle any +// errors coming back after an attempt is made, and to query the switch about +// the status of the attempt. +type duplicateHTLCAttemptInfo struct { + // attemptID is the unique ID used for this attempt. + attemptID uint64 + + // sessionKey is the ephemeral key used for this attempt. + sessionKey [btcec.PrivKeyBytesLen]byte + + // route is the route attempted to send the HTLC. + route Route +} + +// fetchDuplicatePaymentStatus fetches the payment status of the payment. If +// the payment isn't found, it will return error `ErrPaymentNotInitiated`. +func fetchDuplicatePaymentStatus(bucket kvdb.RBucket) (PaymentStatus, error) { + if bucket.Get(duplicatePaymentSettleInfoKey) != nil { + return StatusSucceeded, nil + } + + if bucket.Get(duplicatePaymentFailInfoKey) != nil { + return StatusFailed, nil + } + + if bucket.Get(duplicatePaymentCreationInfoKey) != nil { + return StatusInFlight, nil + } + + return 0, ErrPaymentNotInitiated +} + +func deserializeDuplicateHTLCAttemptInfo(r io.Reader) ( + *duplicateHTLCAttemptInfo, error) { + + a := &duplicateHTLCAttemptInfo{} + err := ReadElements(r, &a.attemptID, &a.sessionKey) + if err != nil { + return nil, err + } + a.route, err = DeserializeRoute(r) + if err != nil { + return nil, err + } + + return a, nil +} + +func deserializeDuplicatePaymentCreationInfo(r io.Reader) ( + *PaymentCreationInfo, error) { + + var scratch [8]byte + + c := &PaymentCreationInfo{} + + if _, err := io.ReadFull(r, c.PaymentIdentifier[:]); err != nil { + return nil, err + } + + if _, err := io.ReadFull(r, scratch[:]); err != nil { + return nil, err + } + c.Value = lnwire.MilliSatoshi(byteOrder.Uint64(scratch[:])) + + if _, err := io.ReadFull(r, scratch[:]); err != nil { + return nil, err + } + c.CreationTime = time.Unix(int64(byteOrder.Uint64(scratch[:])), 0) + + if _, err := io.ReadFull(r, scratch[:4]); err != nil { + return nil, err + } + + reqLen := byteOrder.Uint32(scratch[:4]) + payReq := make([]byte, reqLen) + if reqLen > 0 { + if _, err := io.ReadFull(r, payReq); err != nil { + return nil, err + } + } + c.PaymentRequest = payReq + + return c, nil +} + +func fetchDuplicatePayment(bucket kvdb.RBucket) (*MPPayment, error) { + seqBytes := bucket.Get(duplicatePaymentSequenceKey) + if seqBytes == nil { + return nil, fmt.Errorf("sequence number not found") + } + + sequenceNum := binary.BigEndian.Uint64(seqBytes) + + // Get the payment status. + paymentStatus, err := fetchDuplicatePaymentStatus(bucket) + if err != nil { + return nil, err + } + + // Get the PaymentCreationInfo. + b := bucket.Get(duplicatePaymentCreationInfoKey) + if b == nil { + return nil, fmt.Errorf("creation info not found") + } + + r := bytes.NewReader(b) + creationInfo, err := deserializeDuplicatePaymentCreationInfo(r) + if err != nil { + return nil, err + } + + // Get failure reason if available. + var failureReason *FailureReason + b = bucket.Get(duplicatePaymentFailInfoKey) + if b != nil { + reason := FailureReason(b[0]) + failureReason = &reason + } + + payment := &MPPayment{ + SequenceNum: sequenceNum, + Info: creationInfo, + FailureReason: failureReason, + Status: paymentStatus, + } + + // Get the HTLCAttemptInfo. It can be absent. + b = bucket.Get(duplicatePaymentAttemptInfoKey) + if b != nil { + r = bytes.NewReader(b) + attempt, err := deserializeDuplicateHTLCAttemptInfo(r) + if err != nil { + return nil, err + } + + htlc := HTLCAttempt{ + HTLCAttemptInfo: HTLCAttemptInfo{ + AttemptID: attempt.attemptID, + Route: attempt.route, + sessionKey: attempt.sessionKey, + }, + } + + // Get the payment preimage. This is only found for + // successful payments. + b = bucket.Get(duplicatePaymentSettleInfoKey) + if b != nil { + var preimg lntypes.Preimage + copy(preimg[:], b) + + htlc.Settle = &HTLCSettleInfo{ + Preimage: preimg, + SettleTime: time.Time{}, + } + } else { + // Otherwise the payment must have failed. + htlc.Failure = &HTLCFailInfo{ + FailTime: time.Time{}, + } + } + + payment.HTLCs = []HTLCAttempt{htlc} + } + + return payment, nil +} + +func fetchDuplicatePayments(paymentHashBucket kvdb.RBucket) ([]*MPPayment, + error) { + + var payments []*MPPayment + + // For older versions of lnd, duplicate payments to a payment has was + // possible. These will be found in a sub-bucket indexed by their + // sequence number if available. + dup := paymentHashBucket.NestedReadBucket(duplicatePaymentsBucket) + if dup == nil { + return nil, nil + } + + err := dup.ForEach(func(k, v []byte) error { + subBucket := dup.NestedReadBucket(k) + if subBucket == nil { + // We one bucket for each duplicate to be found. + return fmt.Errorf("non bucket element" + + "in duplicate bucket") + } + + p, err := fetchDuplicatePayment(subBucket) + if err != nil { + return err + } + + payments = append(payments, p) + + return nil + }) + if err != nil { + return nil, err + } + + return payments, nil +} diff --git a/payments/db/migration1/kv_store.go b/payments/db/migration1/kv_store.go new file mode 100644 index 000000000..51bf005db --- /dev/null +++ b/payments/db/migration1/kv_store.go @@ -0,0 +1,2120 @@ +package migration1 + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "sort" + "sync" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // paymentSeqBlockSize is the block size used when we batch allocate + // payment sequences for future payments. + paymentSeqBlockSize = 1000 + + // paymentProgressLogInterval is the interval we use limiting the + // logging output of payment processing. + paymentProgressLogInterval = 30 * time.Second +) + +//nolint:ll +var ( + // paymentsRootBucket is the name of the top-level bucket within the + // database that stores all data related to payments. Within this + // bucket, each payment hash its own sub-bucket keyed by its payment + // hash. + // + // Bucket hierarchy: + // + // root-bucket + // | + // |-- + // | |--sequence-key: + // | |--creation-info-key: + // | |--fail-info-key: <(optional) fail info> + // | | + // | |--payment-htlcs-bucket (shard-bucket) + // | | | + // | | |-- ai: + // | | |-- si: <(optional) settle info> + // | | |-- fi: <(optional) fail info> + // | | | + // | | ... + // | | + // | | + // | |--duplicate-bucket (only for old, completed payments) + // | | + // | |-- + // | | |--sequence-key: + // | | |--creation-info-key: + // | | |--ai: + // | | |--si: + // | | |--fi: + // | | + // | |-- + // | | | + // | ... ... + // | + // |-- + // | | + // | ... + // ... + // + paymentsRootBucket = []byte("payments-root-bucket") + + // paymentSequenceKey is a key used in the payment's sub-bucket to + // store the sequence number of the payment. + paymentSequenceKey = []byte("payment-sequence-key") + + // paymentCreationInfoKey is a key used in the payment's sub-bucket to + // store the creation info of the payment. + paymentCreationInfoKey = []byte("payment-creation-info") + + // paymentHtlcsBucket is a bucket where we'll store the information + // about the HTLCs that were attempted for a payment. + paymentHtlcsBucket = []byte("payment-htlcs-bucket") + + // htlcAttemptInfoKey is the key used as the prefix of an HTLC attempt + // to store the info about the attempt that was done for the HTLC in + // question. The HTLC attempt ID is concatenated at the end. + htlcAttemptInfoKey = []byte("ai") + + // htlcSettleInfoKey is the key used as the prefix of an HTLC attempt + // settle info, if any. The HTLC attempt ID is concatenated at the end. + htlcSettleInfoKey = []byte("si") + + // htlcFailInfoKey is the key used as the prefix of an HTLC attempt + // failure information, if any.The HTLC attempt ID is concatenated at + // the end. + htlcFailInfoKey = []byte("fi") + + // paymentFailInfoKey is a key used in the payment's sub-bucket to + // store information about the reason a payment failed. + paymentFailInfoKey = []byte("payment-fail-info") + + // paymentsIndexBucket is the name of the top-level bucket within the + // database that stores an index of payment sequence numbers to its + // payment hash. + // payments-sequence-index-bucket + // |--: + // |--... + // |--: + paymentsIndexBucket = []byte("payments-index-bucket") +) + +// KVStore implements persistence for payments and payment attempts. +type KVStore struct { + // Sequence management for the kv store. + seqMu sync.Mutex + currSeq uint64 + storedSeq uint64 + + // db is the underlying database implementation. + db kvdb.Backend +} + +// A compile-time constraint to ensure KVStore implements DB. +var _ DB = (*KVStore)(nil) + +// NewKVStore creates a new KVStore for payments. +func NewKVStore(db kvdb.Backend, + options ...OptionModifier) (*KVStore, error) { + + opts := DefaultOptions() + for _, applyOption := range options { + applyOption(opts) + } + + if !opts.NoMigration { + if err := initKVStore(db); err != nil { + return nil, err + } + } + + return &KVStore{ + db: db, + }, nil +} + +// paymentsTopLevelBuckets is a list of top-level buckets that are used for +// the payments database when using the kv store. +var paymentsTopLevelBuckets = [][]byte{ + paymentsRootBucket, + paymentsIndexBucket, +} + +// initKVStore creates and initializes the top-level buckets for the payment db. +func initKVStore(db kvdb.Backend) error { + err := kvdb.Update(db, func(tx kvdb.RwTx) error { + for _, tlb := range paymentsTopLevelBuckets { + if _, err := tx.CreateTopLevelBucket(tlb); err != nil { + return err + } + } + + return nil + }, func() {}) + if err != nil { + return fmt.Errorf("unable to create new payments db: %w", err) + } + + return nil +} + +// InitPayment checks or records the given PaymentCreationInfo with the DB, +// making sure it does not already exist as an in-flight payment. When this +// method returns successfully, the payment is guaranteed to be in the InFlight +// state. +func (p *KVStore) InitPayment(_ context.Context, paymentHash lntypes.Hash, + info *PaymentCreationInfo) error { + + // Obtain a new sequence number for this payment. This is used + // to sort the payments in order of creation, and also acts as + // a unique identifier for each payment. + sequenceNum, err := p.nextPaymentSequence() + if err != nil { + return err + } + + var b bytes.Buffer + if err := serializePaymentCreationInfo(&b, info); err != nil { + return err + } + infoBytes := b.Bytes() + + var updateErr error + err = kvdb.Batch(p.db, func(tx kvdb.RwTx) error { + // Reset the update error, to avoid carrying over an error + // from a previous execution of the batched db transaction. + updateErr = nil + + prefetchPayment(tx, paymentHash) + bucket, err := createPaymentBucket(tx, paymentHash) + if err != nil { + return err + } + + // Get the existing status of this payment, if any. + paymentStatus, err := fetchPaymentStatus(bucket) + + switch { + // If no error is returned, it means we already have this + // payment. We'll check the status to decide whether we allow + // retrying the payment or return a specific error. + case err == nil: + if err := paymentStatus.initializable(); err != nil { + updateErr = err + return nil + } + + // Otherwise, if the error is not `ErrPaymentNotInitiated`, + // we'll return the error. + case !errors.Is(err, ErrPaymentNotInitiated): + return err + } + + // Before we set our new sequence number, we check whether this + // payment has a previously set sequence number and remove its + // index entry if it exists. This happens in the case where we + // have a previously attempted payment which was left in a state + // where we can retry. + seqBytes := bucket.Get(paymentSequenceKey) + if seqBytes != nil { + indexBucket := tx.ReadWriteBucket(paymentsIndexBucket) + if err := indexBucket.Delete(seqBytes); err != nil { + return err + } + } + + // Once we have obtained a sequence number, we add an entry + // to our index bucket which will map the sequence number to + // our payment identifier. + err = createPaymentIndexEntry( + tx, sequenceNum, info.PaymentIdentifier, + ) + if err != nil { + return err + } + + err = bucket.Put(paymentSequenceKey, sequenceNum) + if err != nil { + return err + } + + // Add the payment info to the bucket, which contains the + // static information for this payment + err = bucket.Put(paymentCreationInfoKey, infoBytes) + if err != nil { + return err + } + + // We'll delete any lingering HTLCs to start with, in case we + // are initializing a payment that was attempted earlier, but + // left in a state where we could retry. + err = bucket.DeleteNestedBucket(paymentHtlcsBucket) + if err != nil && !errors.Is(err, kvdb.ErrBucketNotFound) { + return err + } + + // Also delete any lingering failure info now that we are + // re-attempting. + return bucket.Delete(paymentFailInfoKey) + }) + if err != nil { + return fmt.Errorf("unable to init payment: %w", err) + } + + return updateErr +} + +// DeleteFailedAttempts deletes all failed htlcs for a payment. +func (p *KVStore) DeleteFailedAttempts(ctx context.Context, + hash lntypes.Hash) error { + + const failedHtlcsOnly = true + err := p.DeletePayment(ctx, hash, failedHtlcsOnly) + if err != nil { + return err + } + + return nil +} + +// paymentIndexTypeHash is a payment index type which indicates that we have +// created an index of payment sequence number to payment hash. +type paymentIndexType uint8 + +// paymentIndexTypeHash is a payment index type which indicates that we have +// created an index of payment sequence number to payment hash. +const paymentIndexTypeHash paymentIndexType = 0 + +// createPaymentIndexEntry creates a payment hash typed index for a payment. The +// index produced contains a payment index type (which can be used in future to +// signal different payment index types) and the payment identifier. +func createPaymentIndexEntry(tx kvdb.RwTx, sequenceNumber []byte, + id lntypes.Hash) error { + + var b bytes.Buffer + if err := WriteElements(&b, paymentIndexTypeHash, id[:]); err != nil { + return err + } + + indexes := tx.ReadWriteBucket(paymentsIndexBucket) + + return indexes.Put(sequenceNumber, b.Bytes()) +} + +// deserializePaymentIndex deserializes a payment index entry. This function +// currently only supports deserialization of payment hash indexes, and will +// fail for other types. +func deserializePaymentIndex(r io.Reader) (lntypes.Hash, error) { + var ( + indexType paymentIndexType + paymentHash []byte + ) + + if err := ReadElements(r, &indexType, &paymentHash); err != nil { + return lntypes.Hash{}, err + } + + // While we only have on payment index type, we do not need to use our + // index type to deserialize the index. However, we sanity check that + // this type is as expected, since we had to read it out anyway. + if indexType != paymentIndexTypeHash { + return lntypes.Hash{}, fmt.Errorf("unknown payment index "+ + "type: %v", indexType) + } + + hash, err := lntypes.MakeHash(paymentHash) + if err != nil { + return lntypes.Hash{}, err + } + + return hash, nil +} + +// RegisterAttempt atomically records the provided HTLCAttemptInfo to the +// DB. +func (p *KVStore) RegisterAttempt(_ context.Context, paymentHash lntypes.Hash, + attempt *HTLCAttemptInfo) (*MPPayment, error) { + + // Serialize the information before opening the db transaction. + var a bytes.Buffer + err := serializeHTLCAttemptInfo(&a, attempt) + if err != nil { + return nil, err + } + htlcInfoBytes := a.Bytes() + + htlcIDBytes := make([]byte, 8) + binary.BigEndian.PutUint64(htlcIDBytes, attempt.AttemptID) + + var payment *MPPayment + err = kvdb.Batch(p.db, func(tx kvdb.RwTx) error { + prefetchPayment(tx, paymentHash) + bucket, err := fetchPaymentBucketUpdate(tx, paymentHash) + if err != nil { + return err + } + + payment, err = fetchPayment(bucket) + if err != nil { + return err + } + + // Check if registering a new attempt is allowed. + if err := payment.Registrable(); err != nil { + return err + } + + // Verify the attempt is compatible with the existing payment. + if err := verifyAttempt(payment, attempt); err != nil { + return err + } + + htlcsBucket, err := bucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + err = htlcsBucket.Put( + htlcBucketKey(htlcAttemptInfoKey, htlcIDBytes), + htlcInfoBytes, + ) + if err != nil { + return err + } + + // Retrieve attempt info for the notification. + payment, err = fetchPayment(bucket) + + return err + }) + if err != nil { + return nil, err + } + + return payment, err +} + +// SettleAttempt marks the given attempt settled with the preimage. If this is +// a multi shard payment, this might implicitly mean that the full payment +// succeeded. +// +// After invoking this method, InitPayment should always return an error to +// prevent us from making duplicate payments to the same payment hash. The +// provided preimage is atomically saved to the DB for record keeping. +func (p *KVStore) SettleAttempt(_ context.Context, hash lntypes.Hash, + attemptID uint64, settleInfo *HTLCSettleInfo) (*MPPayment, error) { + + var b bytes.Buffer + if err := serializeHTLCSettleInfo(&b, settleInfo); err != nil { + return nil, err + } + settleBytes := b.Bytes() + + return p.updateHtlcKey(hash, attemptID, htlcSettleInfoKey, settleBytes) +} + +// FailAttempt marks the given payment attempt failed. +func (p *KVStore) FailAttempt(_ context.Context, hash lntypes.Hash, + attemptID uint64, failInfo *HTLCFailInfo) (*MPPayment, error) { + + var b bytes.Buffer + if err := serializeHTLCFailInfo(&b, failInfo); err != nil { + return nil, err + } + failBytes := b.Bytes() + + return p.updateHtlcKey(hash, attemptID, htlcFailInfoKey, failBytes) +} + +// updateHtlcKey updates a database key for the specified htlc. +func (p *KVStore) updateHtlcKey(paymentHash lntypes.Hash, + attemptID uint64, key, value []byte) (*MPPayment, error) { + + aid := make([]byte, 8) + binary.BigEndian.PutUint64(aid, attemptID) + + var payment *MPPayment + err := kvdb.Batch(p.db, func(tx kvdb.RwTx) error { + payment = nil + + prefetchPayment(tx, paymentHash) + bucket, err := fetchPaymentBucketUpdate(tx, paymentHash) + if err != nil { + return err + } + + p, err := fetchPayment(bucket) + if err != nil { + return err + } + + // We can only update keys of in-flight payments. We allow + // updating keys even if the payment has reached a terminal + // condition, since the HTLC outcomes must still be updated. + if err := p.Status.updatable(); err != nil { + return err + } + + htlcsBucket := bucket.NestedReadWriteBucket(paymentHtlcsBucket) + if htlcsBucket == nil { + return fmt.Errorf("htlcs bucket not found") + } + + attemptKey := htlcBucketKey(htlcAttemptInfoKey, aid) + if htlcsBucket.Get(attemptKey) == nil { + return fmt.Errorf("HTLC with ID %v not registered", + attemptID) + } + + // Make sure the shard is not already failed or settled. + failKey := htlcBucketKey(htlcFailInfoKey, aid) + if htlcsBucket.Get(failKey) != nil { + return ErrAttemptAlreadyFailed + } + + settleKey := htlcBucketKey(htlcSettleInfoKey, aid) + if htlcsBucket.Get(settleKey) != nil { + return ErrAttemptAlreadySettled + } + + // Add or update the key for this htlc. + err = htlcsBucket.Put(htlcBucketKey(key, aid), value) + if err != nil { + return err + } + + // Retrieve attempt info for the notification. + payment, err = fetchPayment(bucket) + + return err + }) + if err != nil { + return nil, err + } + + return payment, err +} + +// Fail transitions a payment into the Failed state, and records the reason the +// payment failed. After invoking this method, InitPayment should return nil on +// its next call for this payment hash, allowing the switch to make a +// subsequent payment. +func (p *KVStore) Fail(_ context.Context, paymentHash lntypes.Hash, + reason FailureReason) (*MPPayment, error) { + + var ( + updateErr error + payment *MPPayment + ) + err := kvdb.Batch(p.db, func(tx kvdb.RwTx) error { + // Reset the update error, to avoid carrying over an error + // from a previous execution of the batched db transaction. + updateErr = nil + payment = nil + + prefetchPayment(tx, paymentHash) + bucket, err := fetchPaymentBucketUpdate(tx, paymentHash) + if errors.Is(err, ErrPaymentNotInitiated) { + updateErr = ErrPaymentNotInitiated + return nil + } else if err != nil { + return err + } + + // We mark the payment as failed as long as it is known. This + // lets the last attempt to fail with a terminal write its + // failure to the KVStore without synchronizing with + // other attempts. + _, err = fetchPaymentStatus(bucket) + if errors.Is(err, ErrPaymentNotInitiated) { + updateErr = ErrPaymentNotInitiated + return nil + } else if err != nil { + return err + } + + // Put the failure reason in the bucket for record keeping. + v := []byte{byte(reason)} + err = bucket.Put(paymentFailInfoKey, v) + if err != nil { + return err + } + + // Retrieve attempt info for the notification, if available. + payment, err = fetchPayment(bucket) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return nil, err + } + + return payment, updateErr +} + +// FetchPayment returns information about a payment from the database. +func (p *KVStore) FetchPayment(_ context.Context, + paymentHash lntypes.Hash) (*MPPayment, error) { + + var payment *MPPayment + err := kvdb.View(p.db, func(tx kvdb.RTx) error { + prefetchPayment(tx, paymentHash) + bucket, err := fetchPaymentBucket(tx, paymentHash) + if err != nil { + return err + } + + payment, err = fetchPayment(bucket) + + return err + }, func() { + payment = nil + }) + if err != nil { + return nil, err + } + + return payment, nil +} + +// prefetchPayment attempts to prefetch as much of the payment as possible to +// reduce DB roundtrips. +func prefetchPayment(tx kvdb.RTx, paymentHash lntypes.Hash) { + rb := kvdb.RootBucket(tx) + kvdb.Prefetch( + rb, + []string{ + // Prefetch all keys in the payment's bucket. + string(paymentsRootBucket), + string(paymentHash[:]), + }, + []string{ + // Prefetch all keys in the payment's htlc bucket. + string(paymentsRootBucket), + string(paymentHash[:]), + string(paymentHtlcsBucket), + }, + ) +} + +// createPaymentBucket creates or fetches the sub-bucket assigned to this +// payment hash. +func createPaymentBucket(tx kvdb.RwTx, paymentHash lntypes.Hash) ( + kvdb.RwBucket, error) { + + payments, err := tx.CreateTopLevelBucket(paymentsRootBucket) + if err != nil { + return nil, err + } + + return payments.CreateBucketIfNotExists(paymentHash[:]) +} + +// fetchPaymentBucket fetches the sub-bucket assigned to this payment hash. If +// the bucket does not exist, it returns ErrPaymentNotInitiated. +func fetchPaymentBucket(tx kvdb.RTx, paymentHash lntypes.Hash) ( + kvdb.RBucket, error) { + + payments := tx.ReadBucket(paymentsRootBucket) + if payments == nil { + return nil, ErrPaymentNotInitiated + } + + bucket := payments.NestedReadBucket(paymentHash[:]) + if bucket == nil { + return nil, ErrPaymentNotInitiated + } + + return bucket, nil +} + +// fetchPaymentBucketUpdate is identical to fetchPaymentBucket, but it returns a +// bucket that can be written to. +func fetchPaymentBucketUpdate(tx kvdb.RwTx, paymentHash lntypes.Hash) ( + kvdb.RwBucket, error) { + + payments := tx.ReadWriteBucket(paymentsRootBucket) + if payments == nil { + return nil, ErrPaymentNotInitiated + } + + bucket := payments.NestedReadWriteBucket(paymentHash[:]) + if bucket == nil { + return nil, ErrPaymentNotInitiated + } + + return bucket, nil +} + +// nextPaymentSequence returns the next sequence number to store for a new +// payment. +func (p *KVStore) nextPaymentSequence() ([]byte, error) { + p.seqMu.Lock() + defer p.seqMu.Unlock() + + // Set a new upper bound in the DB every 1000 payments to avoid + // conflicts on the sequence when using etcd. + if p.currSeq == p.storedSeq { + var currPaymentSeq, newUpperBound uint64 + if err := kvdb.Update(p.db, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + currPaymentSeq = paymentsBucket.Sequence() + newUpperBound = currPaymentSeq + paymentSeqBlockSize + + return paymentsBucket.SetSequence(newUpperBound) + }, func() {}); err != nil { + return nil, err + } + + // We lazy initialize the cached currPaymentSeq here using the + // first nextPaymentSequence() call. This if statement will auto + // initialize our stored currPaymentSeq, since by default both + // this variable and storedPaymentSeq are zero which in turn + // will have us fetch the current values from the DB. + if p.currSeq == 0 { + p.currSeq = currPaymentSeq + } + + p.storedSeq = newUpperBound + } + + p.currSeq++ + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, p.currSeq) + + return b, nil +} + +// fetchPaymentStatus fetches the payment status of the payment. If the payment +// isn't found, it will return error `ErrPaymentNotInitiated`. +func fetchPaymentStatus(bucket kvdb.RBucket) (PaymentStatus, error) { + // Creation info should be set for all payments, regardless of state. + // If not, it is unknown. + if bucket.Get(paymentCreationInfoKey) == nil { + return 0, ErrPaymentNotInitiated + } + + payment, err := fetchPayment(bucket) + if err != nil { + return 0, err + } + + return payment.Status, nil +} + +// FetchInFlightPayments returns all payments with status InFlight. +func (p *KVStore) FetchInFlightPayments(_ context.Context) ([]*MPPayment, + error) { + + var ( + inFlights []*MPPayment + start = time.Now() + lastLogTime = time.Now() + processedCount int + ) + + err := kvdb.View(p.db, func(tx kvdb.RTx) error { + payments := tx.ReadBucket(paymentsRootBucket) + if payments == nil { + return nil + } + + return payments.ForEach(func(k, _ []byte) error { + bucket := payments.NestedReadBucket(k) + if bucket == nil { + return fmt.Errorf("non bucket element") + } + + p, err := fetchPayment(bucket) + if err != nil { + return err + } + + processedCount++ + if time.Since(lastLogTime) >= + paymentProgressLogInterval { + + log.Debugf("Scanning inflight payments "+ + "(in progress), processed %d, last "+ + "processed payment: %v", processedCount, + p.Info) + + lastLogTime = time.Now() + } + + // Skip the payment if it's terminated. + if p.Terminated() { + return nil + } + + inFlights = append(inFlights, p) + + return nil + }) + }, func() { + inFlights = nil + }) + if err != nil { + return nil, err + } + + elapsed := time.Since(start) + log.Debugf("Completed scanning for inflight payments: "+ + "total_processed=%d, found_inflight=%d, elapsed=%v", + processedCount, len(inFlights), + elapsed.Round(time.Millisecond)) + + return inFlights, nil +} + +// htlcBucketKey creates a composite key from prefix and id where the result is +// simply the two concatenated. +func htlcBucketKey(prefix, id []byte) []byte { + key := make([]byte, len(prefix)+len(id)) + copy(key, prefix) + copy(key[len(prefix):], id) + + return key +} + +// FetchPayments returns all sent payments found in the DB. +func (p *KVStore) FetchPayments() ([]*MPPayment, error) { + var payments []*MPPayment + + err := kvdb.View(p.db, func(tx kvdb.RTx) error { + paymentsBucket := tx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + return nil + } + + return paymentsBucket.ForEach(func(k, v []byte) error { + bucket := paymentsBucket.NestedReadBucket(k) + if bucket == nil { + // We only expect sub-buckets to be found in + // this top-level bucket. + return fmt.Errorf("non bucket element in " + + "payments bucket") + } + + p, err := fetchPayment(bucket) + if err != nil { + return err + } + + payments = append(payments, p) + + // For older versions of lnd, duplicate payments to a + // payment has was possible. These will be found in a + // sub-bucket indexed by their sequence number if + // available. + duplicatePayments, err := fetchDuplicatePayments(bucket) + if err != nil { + return err + } + + payments = append(payments, duplicatePayments...) + + return nil + }) + }, func() { + payments = nil + }) + if err != nil { + return nil, err + } + + // Before returning, sort the payments by their sequence number. + sort.Slice(payments, func(i, j int) bool { + return payments[i].SequenceNum < payments[j].SequenceNum + }) + + return payments, nil +} + +func fetchCreationInfo(bucket kvdb.RBucket) (*PaymentCreationInfo, error) { + b := bucket.Get(paymentCreationInfoKey) + if b == nil { + return nil, fmt.Errorf("creation info not found") + } + + r := bytes.NewReader(b) + + return deserializePaymentCreationInfo(r) +} + +func fetchPayment(bucket kvdb.RBucket) (*MPPayment, error) { + seqBytes := bucket.Get(paymentSequenceKey) + if seqBytes == nil { + return nil, fmt.Errorf("sequence number not found") + } + + sequenceNum := binary.BigEndian.Uint64(seqBytes) + + // Get the PaymentCreationInfo. + creationInfo, err := fetchCreationInfo(bucket) + if err != nil { + return nil, err + } + + var htlcs []HTLCAttempt + htlcsBucket := bucket.NestedReadBucket(paymentHtlcsBucket) + if htlcsBucket != nil { + // Get the payment attempts. This can be empty. + htlcs, err = fetchHtlcAttempts(htlcsBucket) + if err != nil { + return nil, err + } + } + + // Get failure reason if available. + var failureReason *FailureReason + b := bucket.Get(paymentFailInfoKey) + if b != nil { + reason := FailureReason(b[0]) + failureReason = &reason + } + + // Create a new payment. + payment := &MPPayment{ + SequenceNum: sequenceNum, + Info: creationInfo, + HTLCs: htlcs, + FailureReason: failureReason, + } + + // Set its state and status. + if err := payment.setState(); err != nil { + return nil, err + } + + return payment, nil +} + +// fetchHtlcAttempts retrieves all htlc attempts made for the payment found in +// the given bucket. +func fetchHtlcAttempts(bucket kvdb.RBucket) ([]HTLCAttempt, error) { + htlcsMap := make(map[uint64]*HTLCAttempt) + + attemptInfoCount := 0 + err := bucket.ForEach(func(k, v []byte) error { + aid := byteOrder.Uint64(k[len(k)-8:]) + + if _, ok := htlcsMap[aid]; !ok { + htlcsMap[aid] = &HTLCAttempt{} + } + + var err error + switch { + case bytes.HasPrefix(k, htlcAttemptInfoKey): + attemptInfo, err := readHtlcAttemptInfo(v) + if err != nil { + return err + } + + attemptInfo.AttemptID = aid + htlcsMap[aid].HTLCAttemptInfo = *attemptInfo + attemptInfoCount++ + + case bytes.HasPrefix(k, htlcSettleInfoKey): + htlcsMap[aid].Settle, err = readHtlcSettleInfo(v) + if err != nil { + return err + } + + case bytes.HasPrefix(k, htlcFailInfoKey): + htlcsMap[aid].Failure, err = readHtlcFailInfo(v) + if err != nil { + return err + } + + default: + return fmt.Errorf("unknown htlc attempt key") + } + + return nil + }) + if err != nil { + return nil, err + } + + // Sanity check that all htlcs have an attempt info. + if attemptInfoCount != len(htlcsMap) { + return nil, ErrNoAttemptInfo + } + + keys := make([]uint64, len(htlcsMap)) + i := 0 + for k := range htlcsMap { + keys[i] = k + i++ + } + + // Sort HTLC attempts by their attempt ID. This is needed because in the + // DB we store the attempts with keys prefixed by their status which + // changes order (groups them together by status). + sort.Slice(keys, func(i, j int) bool { + return keys[i] < keys[j] + }) + + htlcs := make([]HTLCAttempt, len(htlcsMap)) + for i, key := range keys { + htlcs[i] = *htlcsMap[key] + } + + return htlcs, nil +} + +// readHtlcAttemptInfo reads the payment attempt info for this htlc. +func readHtlcAttemptInfo(b []byte) (*HTLCAttemptInfo, error) { + r := bytes.NewReader(b) + return deserializeHTLCAttemptInfo(r) +} + +// readHtlcSettleInfo reads the settle info for the htlc. If the htlc isn't +// settled, nil is returned. +func readHtlcSettleInfo(b []byte) (*HTLCSettleInfo, error) { + r := bytes.NewReader(b) + return deserializeHTLCSettleInfo(r) +} + +// readHtlcFailInfo reads the failure info for the htlc. If the htlc hasn't +// failed, nil is returned. +func readHtlcFailInfo(b []byte) (*HTLCFailInfo, error) { + r := bytes.NewReader(b) + return deserializeHTLCFailInfo(r) +} + +// fetchFailedHtlcKeys retrieves the bucket keys of all failed HTLCs of a +// payment bucket. +func fetchFailedHtlcKeys(bucket kvdb.RBucket) ([][]byte, error) { + htlcsBucket := bucket.NestedReadBucket(paymentHtlcsBucket) + + var htlcs []HTLCAttempt + var err error + if htlcsBucket != nil { + htlcs, err = fetchHtlcAttempts(htlcsBucket) + if err != nil { + return nil, err + } + } + + // Now iterate though them and save the bucket keys for the failed + // HTLCs. + var htlcKeys [][]byte + for _, h := range htlcs { + if h.Failure == nil { + continue + } + + htlcKeyBytes := make([]byte, 8) + binary.BigEndian.PutUint64(htlcKeyBytes, h.AttemptID) + + htlcKeys = append(htlcKeys, htlcKeyBytes) + } + + return htlcKeys, nil +} + +// QueryPayments is a query to the payments database which is restricted +// to a subset of payments by the payments query, containing an offset +// index and a maximum number of returned payments. +func (p *KVStore) QueryPayments(_ context.Context, + query Query) (Response, error) { + + var resp Response + + if err := kvdb.View(p.db, func(tx kvdb.RTx) error { + // Get the root payments bucket. + paymentsBucket := tx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + return nil + } + + // Get the index bucket which maps sequence number -> payment + // hash and duplicate bool. If we have a payments bucket, we + // should have an indexes bucket as well. + indexes := tx.ReadBucket(paymentsIndexBucket) + if indexes == nil { + return fmt.Errorf("index bucket does not exist") + } + + // accumulatePayments gets payments with the sequence number + // and hash provided and adds them to our list of payments if + // they meet the criteria of our query. It returns the number + // of payments that were added. + accumulatePayments := func(sequenceKey, hash []byte) (bool, + error) { + + r := bytes.NewReader(hash) + paymentHash, err := deserializePaymentIndex(r) + if err != nil { + return false, err + } + + payment, err := fetchPaymentWithSequenceNumber( + tx, paymentHash, sequenceKey, + ) + if err != nil { + return false, err + } + + // To keep compatibility with the old API, we only + // return non-succeeded payments if requested. + if payment.Status != StatusSucceeded && + !query.IncludeIncomplete { + + return false, err + } + + // Get the creation time in Unix seconds, this always + // rounds down the nanoseconds to full seconds. + createTime := payment.Info.CreationTime.Unix() + + // Skip any payments that were created before the + // specified time. + if createTime < query.CreationDateStart { + return false, nil + } + + // Skip any payments that were created after the + // specified time. + if query.CreationDateEnd != 0 && + createTime > query.CreationDateEnd { + + return false, nil + } + + // At this point, we've exhausted the offset, so we'll + // begin collecting invoices found within the range. + resp.Payments = append(resp.Payments, payment) + + return true, nil + } + + // Create a paginator which reads from our sequence index bucket + // with the parameters provided by the payments query. + paginator := channeldb.NewPaginator( + indexes.ReadCursor(), query.Reversed, query.IndexOffset, + query.MaxPayments, + ) + + // Run a paginated query, adding payments to our response. + if err := paginator.Query(accumulatePayments); err != nil { + return err + } + + // Counting the total number of payments is expensive, since we + // literally have to traverse the cursor linearly, which can + // take quite a while. So it's an optional query parameter. + if query.CountTotal { + var ( + totalPayments uint64 + err error + ) + countFn := func(_, _ []byte) error { + totalPayments++ + + return nil + } + + // In non-boltdb database backends, there's a faster + // ForAll query that allows for batch fetching items. + fastBucket, ok := indexes.(kvdb.ExtendedRBucket) + if ok { + err = fastBucket.ForAll(countFn) + } else { + err = indexes.ForEach(countFn) + } + if err != nil { + return fmt.Errorf("error counting payments: %w", + err) + } + + resp.TotalCount = totalPayments + } + + return nil + }, func() { + resp = Response{} + }); err != nil { + return resp, err + } + + // Need to swap the payments slice order if reversed order. + if query.Reversed { + for l, r := 0, len(resp.Payments)-1; l < r; l, r = l+1, r-1 { + resp.Payments[l], resp.Payments[r] = + resp.Payments[r], resp.Payments[l] + } + } + + // Set the first and last index of the returned payments so that the + // caller can resume from this point later on. + if len(resp.Payments) > 0 { + resp.FirstIndexOffset = resp.Payments[0].SequenceNum + resp.LastIndexOffset = + resp.Payments[len(resp.Payments)-1].SequenceNum + } + + return resp, nil +} + +// fetchPaymentWithSequenceNumber get the payment which matches the payment hash +// *and* sequence number provided from the database. This is required because +// we previously had more than one payment per hash, so we have multiple indexes +// pointing to a single payment; we want to retrieve the correct one. +func fetchPaymentWithSequenceNumber(tx kvdb.RTx, paymentHash lntypes.Hash, + sequenceNumber []byte) (*MPPayment, error) { + + // We can now lookup the payment keyed by its hash in + // the payments root bucket. + bucket, err := fetchPaymentBucket(tx, paymentHash) + if err != nil { + return nil, err + } + + // A single payment hash can have multiple payments associated with it. + // We lookup our sequence number first, to determine whether this is + // the payment we are actually looking for. + seqBytes := bucket.Get(paymentSequenceKey) + if seqBytes == nil { + return nil, ErrNoSequenceNumber + } + + // If this top level payment has the sequence number we are looking for, + // return it. + if bytes.Equal(seqBytes, sequenceNumber) { + return fetchPayment(bucket) + } + + // If we were not looking for the top level payment, we are looking for + // one of our duplicate payments. We need to iterate through the seq + // numbers in this bucket to find the correct payments. If we do not + // find a duplicate payments bucket here, something is wrong. + dup := bucket.NestedReadBucket(duplicatePaymentsBucket) + if dup == nil { + return nil, ErrNoDuplicateBucket + } + + var duplicatePayment *MPPayment + err = dup.ForEach(func(k, v []byte) error { + subBucket := dup.NestedReadBucket(k) + if subBucket == nil { + // We one bucket for each duplicate to be found. + return ErrNoDuplicateNestedBucket + } + + seqBytes := subBucket.Get(duplicatePaymentSequenceKey) + if seqBytes == nil { + return ErrNoDuplicateSequenceNumber + } + + // If this duplicate payment is not the sequence number we are + // looking for, we can continue. + if !bytes.Equal(seqBytes, sequenceNumber) { + return nil + } + + duplicatePayment, err = fetchDuplicatePayment(subBucket) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return nil, err + } + + // If none of the duplicate payments matched our sequence number, we + // failed to find the payment with this sequence number; something is + // wrong. + if duplicatePayment == nil { + return nil, ErrDuplicateNotFound + } + + return duplicatePayment, nil +} + +// DeletePayment deletes a payment from the DB given its payment hash. If +// failedHtlcsOnly is set, only failed HTLC attempts of the payment will be +// deleted. +func (p *KVStore) DeletePayment(_ context.Context, paymentHash lntypes.Hash, + failedHtlcsOnly bool) error { + + return kvdb.Update(p.db, func(tx kvdb.RwTx) error { + payments := tx.ReadWriteBucket(paymentsRootBucket) + if payments == nil { + return nil + } + + bucket := payments.NestedReadWriteBucket(paymentHash[:]) + if bucket == nil { + return fmt.Errorf("non bucket element in payments " + + "bucket") + } + + // If the status is InFlight, we cannot safely delete + // the payment information, so we return early. + paymentStatus, err := fetchPaymentStatus(bucket) + if err != nil { + return err + } + + // If the payment has inflight HTLCs, we cannot safely delete + // the payment information, so we return an error. + if err := paymentStatus.removable(); err != nil { + return fmt.Errorf("payment '%v' has inflight HTLCs"+ + "and therefore cannot be deleted: %w", + paymentHash.String(), err) + } + + // Delete the failed HTLC attempts we found. + if failedHtlcsOnly { + toDelete, err := fetchFailedHtlcKeys(bucket) + if err != nil { + return err + } + + htlcsBucket := bucket.NestedReadWriteBucket( + paymentHtlcsBucket, + ) + + for _, htlcID := range toDelete { + err = htlcsBucket.Delete( + htlcBucketKey( + htlcAttemptInfoKey, htlcID, + ), + ) + if err != nil { + return err + } + + err = htlcsBucket.Delete( + htlcBucketKey(htlcFailInfoKey, htlcID), + ) + if err != nil { + return err + } + + err = htlcsBucket.Delete( + htlcBucketKey( + htlcSettleInfoKey, htlcID, + ), + ) + if err != nil { + return err + } + } + + return nil + } + + seqNrs, err := fetchSequenceNumbers(bucket) + if err != nil { + return err + } + + err = payments.DeleteNestedBucket(paymentHash[:]) + if err != nil { + return err + } + + indexBucket := tx.ReadWriteBucket(paymentsIndexBucket) + for _, k := range seqNrs { + if err := indexBucket.Delete(k); err != nil { + return err + } + } + + return nil + }, func() {}) +} + +// DeletePayments deletes all completed and failed payments from the DB. If +// failedOnly is set, only failed payments will be considered for deletion. If +// failedHtlcsOnly is set, the payment itself won't be deleted, only failed HTLC +// attempts. The method returns the number of deleted payments, which is always +// 0 if failedHtlcsOnly is set. +func (p *KVStore) DeletePayments(_ context.Context, failedOnly, + failedHtlcsOnly bool) (int, error) { + + var numPayments int + err := kvdb.Update(p.db, func(tx kvdb.RwTx) error { + payments := tx.ReadWriteBucket(paymentsRootBucket) + if payments == nil { + return nil + } + + var ( + // deleteBuckets is the set of payment buckets we need + // to delete. + deleteBuckets [][]byte + + // deleteIndexes is the set of indexes pointing to these + // payments that need to be deleted. + deleteIndexes [][]byte + + // deleteHtlcs maps a payment hash to the HTLC IDs we + // want to delete for that payment. + deleteHtlcs = make(map[lntypes.Hash][][]byte) + ) + err := payments.ForEach(func(k, _ []byte) error { + bucket := payments.NestedReadBucket(k) + if bucket == nil { + // We only expect sub-buckets to be found in + // this top-level bucket. + return fmt.Errorf("non bucket element in " + + "payments bucket") + } + + // If the status is InFlight, we cannot safely delete + // the payment information, so we return early. + paymentStatus, err := fetchPaymentStatus(bucket) + if err != nil { + return err + } + + // If the payment has inflight HTLCs, we cannot safely + // delete the payment information, so we return an nil + // to skip it. + if err := paymentStatus.removable(); err != nil { + return nil + } + + // If we requested to only delete failed payments, we + // can return if this one is not. + if failedOnly && paymentStatus != StatusFailed { + return nil + } + + // If we are only deleting failed HTLCs, fetch them. + if failedHtlcsOnly { + toDelete, err := fetchFailedHtlcKeys(bucket) + if err != nil { + return err + } + + hash, err := lntypes.MakeHash(k) + if err != nil { + return err + } + + deleteHtlcs[hash] = toDelete + + // We return, we are only deleting attempts. + return nil + } + + // Add the bucket to the set of buckets we can delete. + deleteBuckets = append(deleteBuckets, k) + + // Get all the sequence number associated with the + // payment, including duplicates. + seqNrs, err := fetchSequenceNumbers(bucket) + if err != nil { + return err + } + + deleteIndexes = append(deleteIndexes, seqNrs...) + numPayments++ + + return nil + }) + if err != nil { + return err + } + + // Delete the failed HTLC attempts we found. + for hash, htlcIDs := range deleteHtlcs { + bucket := payments.NestedReadWriteBucket(hash[:]) + htlcsBucket := bucket.NestedReadWriteBucket( + paymentHtlcsBucket, + ) + + for _, aid := range htlcIDs { + if err := htlcsBucket.Delete( + htlcBucketKey(htlcAttemptInfoKey, aid), + ); err != nil { + return err + } + + if err := htlcsBucket.Delete( + htlcBucketKey(htlcFailInfoKey, aid), + ); err != nil { + return err + } + + if err := htlcsBucket.Delete( + htlcBucketKey(htlcSettleInfoKey, aid), + ); err != nil { + return err + } + } + } + + for _, k := range deleteBuckets { + if err := payments.DeleteNestedBucket(k); err != nil { + return err + } + } + + // Get our index bucket and delete all indexes pointing to the + // payments we are deleting. + indexBucket := tx.ReadWriteBucket(paymentsIndexBucket) + for _, k := range deleteIndexes { + if err := indexBucket.Delete(k); err != nil { + return err + } + } + + return nil + }, func() { + numPayments = 0 + }) + if err != nil { + return 0, err + } + + return numPayments, nil +} + +// fetchSequenceNumbers fetches all the sequence numbers associated with a +// payment, including those belonging to any duplicate payments. +func fetchSequenceNumbers(paymentBucket kvdb.RBucket) ([][]byte, error) { + seqNum := paymentBucket.Get(paymentSequenceKey) + if seqNum == nil { + return nil, errors.New("expected sequence number") + } + + sequenceNumbers := [][]byte{seqNum} + + // Get the duplicate payments bucket, if it has no duplicates, just + // return early with the payment sequence number. + duplicates := paymentBucket.NestedReadBucket(duplicatePaymentsBucket) + if duplicates == nil { + return sequenceNumbers, nil + } + + // If we do have duplicated, they are keyed by sequence number, so we + // iterate through the duplicates bucket and add them to our set of + // sequence numbers. + if err := duplicates.ForEach(func(k, v []byte) error { + sequenceNumbers = append(sequenceNumbers, k) + return nil + }); err != nil { + return nil, err + } + + return sequenceNumbers, nil +} + +func serializePaymentCreationInfo(w io.Writer, c *PaymentCreationInfo) error { + var scratch [8]byte + + if _, err := w.Write(c.PaymentIdentifier[:]); err != nil { + return err + } + + byteOrder.PutUint64(scratch[:], uint64(c.Value)) + if _, err := w.Write(scratch[:]); err != nil { + return err + } + + if err := serializeTime(w, c.CreationTime); err != nil { + return err + } + + byteOrder.PutUint32(scratch[:4], uint32(len(c.PaymentRequest))) + if _, err := w.Write(scratch[:4]); err != nil { + return err + } + + if _, err := w.Write(c.PaymentRequest); err != nil { + return err + } + + // Any remaining bytes are TLV encoded records. Currently, these are + // only the custom records provided by the user to be sent to the first + // hop. But this can easily be extended with further records by merging + // the records into a single TLV stream. + err := c.FirstHopCustomRecords.SerializeTo(w) + if err != nil { + return err + } + + return nil +} + +func deserializePaymentCreationInfo(r io.Reader) (*PaymentCreationInfo, + error) { + + var scratch [8]byte + + c := &PaymentCreationInfo{} + + if _, err := io.ReadFull(r, c.PaymentIdentifier[:]); err != nil { + return nil, err + } + + if _, err := io.ReadFull(r, scratch[:]); err != nil { + return nil, err + } + c.Value = lnwire.MilliSatoshi(byteOrder.Uint64(scratch[:])) + + creationTime, err := deserializeTime(r) + if err != nil { + return nil, err + } + c.CreationTime = creationTime + + if _, err := io.ReadFull(r, scratch[:4]); err != nil { + return nil, err + } + + reqLen := byteOrder.Uint32(scratch[:4]) + payReq := make([]byte, reqLen) + if reqLen > 0 { + if _, err := io.ReadFull(r, payReq); err != nil { + return nil, err + } + } + c.PaymentRequest = payReq + + // Any remaining bytes are TLV encoded records. Currently, these are + // only the custom records provided by the user to be sent to the first + // hop. But this can easily be extended with further records by merging + // the records into a single TLV stream. + c.FirstHopCustomRecords, err = lnwire.ParseCustomRecordsFrom(r) + if err != nil { + return nil, err + } + + return c, nil +} + +func serializeHTLCAttemptInfo(w io.Writer, a *HTLCAttemptInfo) error { + if err := WriteElements(w, a.sessionKey); err != nil { + return err + } + + if err := SerializeRoute(w, a.Route); err != nil { + return err + } + + if err := serializeTime(w, a.AttemptTime); err != nil { + return err + } + + // If the hash is nil we can just return. + if a.Hash == nil { + return nil + } + + if _, err := w.Write(a.Hash[:]); err != nil { + return err + } + + // Merge the fixed/known records together with the custom records to + // serialize them as a single blob. We can't do this in SerializeRoute + // because we're in the middle of the byte stream there. We can only do + // TLV serialization at the end of the stream, since EOF is allowed for + // a stream if no more data is expected. + producers := []tlv.RecordProducer{ + &a.Route.FirstHopAmount, + } + tlvData, err := lnwire.MergeAndEncode( + producers, nil, a.Route.FirstHopWireCustomRecords, + ) + if err != nil { + return err + } + + if _, err := w.Write(tlvData); err != nil { + return err + } + + return nil +} + +func deserializeHTLCAttemptInfo(r io.Reader) (*HTLCAttemptInfo, error) { + a := &HTLCAttemptInfo{} + err := ReadElements(r, &a.sessionKey) + if err != nil { + return nil, err + } + + a.Route, err = DeserializeRoute(r) + if err != nil { + return nil, err + } + + a.AttemptTime, err = deserializeTime(r) + if err != nil { + return nil, err + } + + hash := lntypes.Hash{} + _, err = io.ReadFull(r, hash[:]) + + switch { + // Older payment attempts wouldn't have the hash set, in which case we + // can just return. + case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): + return a, nil + + case err != nil: + return nil, err + + default: + } + + a.Hash = &hash + + // Read any remaining data (if any) and parse it into the known records + // and custom records. + extraData, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + customRecords, _, _, err := lnwire.ParseAndExtractCustomRecords( + extraData, &a.Route.FirstHopAmount, + ) + if err != nil { + return nil, err + } + + a.Route.FirstHopWireCustomRecords = customRecords + + return a, nil +} + +func serializeHop(w io.Writer, h *Hop) error { + if err := WriteElements(w, + h.PubKeyBytes[:], + h.ChannelID, + h.OutgoingTimeLock, + h.AmtToForward, + ); err != nil { + return err + } + + if err := binary.Write(w, byteOrder, h.LegacyPayload); err != nil { + return err + } + + // For legacy payloads, we don't need to write any TLV records, so + // we'll write a zero indicating the our serialized TLV map has no + // records. + if h.LegacyPayload { + return WriteElements(w, uint32(0)) + } + + // Gather all non-primitive TLV records so that they can be serialized + // as a single blob. + // + // TODO(conner): add migration to unify all fields in a single TLV + // blobs. The split approach will cause headaches down the road as more + // fields are added, which we can avoid by having a single TLV stream + // for all payload fields. + var records []tlv.Record + if h.MPP != nil { + records = append(records, h.MPP.Record()) + } + + // Add blinding point and encrypted data if present. + if h.EncryptedData != nil { + records = append(records, record.NewEncryptedDataRecord( + &h.EncryptedData, + )) + } + + if h.BlindingPoint != nil { + records = append(records, record.NewBlindingPointRecord( + &h.BlindingPoint, + )) + } + + if h.AMP != nil { + records = append(records, h.AMP.Record()) + } + + if h.Metadata != nil { + records = append(records, record.NewMetadataRecord(&h.Metadata)) + } + + if h.TotalAmtMsat != 0 { + totalMsatInt := uint64(h.TotalAmtMsat) + records = append( + records, record.NewTotalAmtMsatBlinded(&totalMsatInt), + ) + } + + // Final sanity check to absolutely rule out custom records that are not + // custom and write into the standard range. + if err := h.CustomRecords.Validate(); err != nil { + return err + } + + // Convert custom records to tlv and add to the record list. + // MapToRecords sorts the list, so adding it here will keep the list + // canonical. + tlvRecords := tlv.MapToRecords(h.CustomRecords) + records = append(records, tlvRecords...) + + // Otherwise, we'll transform our slice of records into a map of the + // raw bytes, then serialize them in-line with a length (number of + // elements) prefix. + mapRecords, err := tlv.RecordsToMap(records) + if err != nil { + return err + } + + numRecords := uint32(len(mapRecords)) + if err := WriteElements(w, numRecords); err != nil { + return err + } + + for recordType, rawBytes := range mapRecords { + if err := WriteElements(w, recordType); err != nil { + return err + } + + if err := wire.WriteVarBytes(w, 0, rawBytes); err != nil { + return err + } + } + + return nil +} + +// maxOnionPayloadSize is the largest Sphinx payload possible, so we don't need +// to read/write a TLV stream larger than this. +const maxOnionPayloadSize = 1300 + +func deserializeHop(r io.Reader) (*Hop, error) { + h := &Hop{} + + var pub []byte + if err := ReadElements(r, &pub); err != nil { + return nil, err + } + copy(h.PubKeyBytes[:], pub) + + if err := ReadElements(r, + &h.ChannelID, &h.OutgoingTimeLock, &h.AmtToForward, + ); err != nil { + return nil, err + } + + // TODO(roasbeef): change field to allow LegacyPayload false to be the + // legacy default? + err := binary.Read(r, byteOrder, &h.LegacyPayload) + if err != nil { + return nil, err + } + + var numElements uint32 + if err := ReadElements(r, &numElements); err != nil { + return nil, err + } + + // If there're no elements, then we can return early. + if numElements == 0 { + return h, nil + } + + tlvMap := make(map[uint64][]byte) + for i := uint32(0); i < numElements; i++ { + var tlvType uint64 + if err := ReadElements(r, &tlvType); err != nil { + return nil, err + } + + rawRecordBytes, err := wire.ReadVarBytes( + r, 0, maxOnionPayloadSize, "tlv", + ) + if err != nil { + return nil, err + } + + tlvMap[tlvType] = rawRecordBytes + } + + // If the MPP type is present, remove it from the generic TLV map and + // parse it back into a proper MPP struct. + // + // TODO(conner): add migration to unify all fields in a single TLV + // blobs. The split approach will cause headaches down the road as more + // fields are added, which we can avoid by having a single TLV stream + // for all payload fields. + mppType := uint64(record.MPPOnionType) + if mppBytes, ok := tlvMap[mppType]; ok { + delete(tlvMap, mppType) + + var ( + mpp = &record.MPP{} + mppRec = mpp.Record() + r = bytes.NewReader(mppBytes) + ) + err := mppRec.Decode(r, uint64(len(mppBytes))) + if err != nil { + return nil, err + } + h.MPP = mpp + } + + // If encrypted data or blinding key are present, remove them from + // the TLV map and parse into proper types. + encryptedDataType := uint64(record.EncryptedDataOnionType) + if data, ok := tlvMap[encryptedDataType]; ok { + delete(tlvMap, encryptedDataType) + h.EncryptedData = data + } + + blindingType := uint64(record.BlindingPointOnionType) + if blindingPoint, ok := tlvMap[blindingType]; ok { + delete(tlvMap, blindingType) + + h.BlindingPoint, err = btcec.ParsePubKey(blindingPoint) + if err != nil { + return nil, fmt.Errorf("invalid blinding point: %w", + err) + } + } + + ampType := uint64(record.AMPOnionType) + if ampBytes, ok := tlvMap[ampType]; ok { + delete(tlvMap, ampType) + + var ( + amp = &record.AMP{} + ampRec = amp.Record() + r = bytes.NewReader(ampBytes) + ) + err := ampRec.Decode(r, uint64(len(ampBytes))) + if err != nil { + return nil, err + } + h.AMP = amp + } + + // If the metadata type is present, remove it from the tlv map and + // populate directly on the hop. + metadataType := uint64(record.MetadataOnionType) + if metadata, ok := tlvMap[metadataType]; ok { + delete(tlvMap, metadataType) + + h.Metadata = metadata + } + + totalAmtMsatType := uint64(record.TotalAmtMsatBlindedType) + if totalAmtMsat, ok := tlvMap[totalAmtMsatType]; ok { + delete(tlvMap, totalAmtMsatType) + + var ( + totalAmtMsatInt uint64 + buf [8]byte + ) + if err := tlv.DTUint64( + bytes.NewReader(totalAmtMsat), + &totalAmtMsatInt, + &buf, + uint64(len(totalAmtMsat)), + ); err != nil { + return nil, err + } + + h.TotalAmtMsat = lnwire.MilliSatoshi(totalAmtMsatInt) + } + + h.CustomRecords = tlvMap + + return h, nil +} + +// SerializeRoute serializes a route. +func SerializeRoute(w io.Writer, r Route) error { + if err := WriteElements(w, + r.TotalTimeLock, r.TotalAmount, r.SourcePubKey[:], + ); err != nil { + return err + } + + if err := WriteElements(w, uint32(len(r.Hops))); err != nil { + return err + } + + for _, h := range r.Hops { + if err := serializeHop(w, h); err != nil { + return err + } + } + + // Any new/extra TLV data is encoded in serializeHTLCAttemptInfo! + + return nil +} + +// DeserializeRoute deserializes a route. +func DeserializeRoute(r io.Reader) (Route, error) { + rt := Route{} + if err := ReadElements(r, + &rt.TotalTimeLock, &rt.TotalAmount, + ); err != nil { + return rt, err + } + + var pub []byte + if err := ReadElements(r, &pub); err != nil { + return rt, err + } + copy(rt.SourcePubKey[:], pub) + + var numHops uint32 + if err := ReadElements(r, &numHops); err != nil { + return rt, err + } + + var hops []*Hop + for i := uint32(0); i < numHops; i++ { + hop, err := deserializeHop(r) + if err != nil { + return rt, err + } + hops = append(hops, hop) + } + rt.Hops = hops + + // Any new/extra TLV data is decoded in deserializeHTLCAttemptInfo! + + return rt, nil +} + +// serializeHTLCSettleInfo serializes the details of a settled htlc. +func serializeHTLCSettleInfo(w io.Writer, s *HTLCSettleInfo) error { + if _, err := w.Write(s.Preimage[:]); err != nil { + return err + } + + if err := serializeTime(w, s.SettleTime); err != nil { + return err + } + + return nil +} + +// deserializeHTLCSettleInfo deserializes the details of a settled htlc. +func deserializeHTLCSettleInfo(r io.Reader) (*HTLCSettleInfo, error) { + s := &HTLCSettleInfo{} + if _, err := io.ReadFull(r, s.Preimage[:]); err != nil { + return nil, err + } + + var err error + s.SettleTime, err = deserializeTime(r) + if err != nil { + return nil, err + } + + return s, nil +} + +// serializeHTLCFailInfo serializes the details of a failed htlc including the +// wire failure. +func serializeHTLCFailInfo(w io.Writer, f *HTLCFailInfo) error { + if err := serializeTime(w, f.FailTime); err != nil { + return err + } + + // Write failure. If there is no failure message, write an empty + // byte slice. + var messageBytes bytes.Buffer + if f.Message != nil { + err := lnwire.EncodeFailureMessage(&messageBytes, f.Message, 0) + if err != nil { + return err + } + } + if err := wire.WriteVarBytes(w, 0, messageBytes.Bytes()); err != nil { + return err + } + + return WriteElements(w, byte(f.Reason), f.FailureSourceIndex) +} + +// deserializeHTLCFailInfo deserializes the details of a failed htlc including +// the wire failure. +func deserializeHTLCFailInfo(r io.Reader) (*HTLCFailInfo, error) { + f := &HTLCFailInfo{} + var err error + f.FailTime, err = deserializeTime(r) + if err != nil { + return nil, err + } + + // Read failure. + failureBytes, err := wire.ReadVarBytes( + r, 0, math.MaxUint16, "failure", + ) + if err != nil { + return nil, err + } + if len(failureBytes) > 0 { + f.Message, err = lnwire.DecodeFailureMessage( + bytes.NewReader(failureBytes), 0, + ) + if err != nil && + !errors.Is(err, lnwire.ErrParsingExtraTLVBytes) { + + return nil, err + } + + // In case we have an invalid TLV stream regarding the extra + // tlv data we still continue with the decoding of the + // HTLCFailInfo. + if errors.Is(err, lnwire.ErrParsingExtraTLVBytes) { + log.Warnf("Failed to decode extra TLV bytes for "+ + "failure message: %v", err) + } + } + + var reason byte + err = ReadElements(r, &reason, &f.FailureSourceIndex) + if err != nil { + return nil, err + } + f.Reason = HTLCFailReason(reason) + + return f, nil +} diff --git a/payments/db/migration1/log.go b/payments/db/migration1/log.go new file mode 100644 index 000000000..52f1f7505 --- /dev/null +++ b/payments/db/migration1/log.go @@ -0,0 +1,32 @@ +package migration1 + +import ( + "github.com/btcsuite/btclog/v2" + "github.com/lightningnetwork/lnd/build" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log btclog.Logger + +// Subsystem defines the logging identifier for this subsystem. +const Subsystem = "PYDB" + +// The default amount of logging is none. +func init() { + UseLogger(build.NewSubLogger(Subsystem, nil)) +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until UseLogger is called. +func DisableLog() { + UseLogger(btclog.Disabled) +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using btclog. +func UseLogger(logger btclog.Logger) { + log = logger +} diff --git a/payments/db/migration1/options.go b/payments/db/migration1/options.go new file mode 100644 index 000000000..382afb26c --- /dev/null +++ b/payments/db/migration1/options.go @@ -0,0 +1,26 @@ +package migration1 + +// StoreOptions holds parameters for the KVStore. +type StoreOptions struct { + // NoMigration allows to open the database in readonly mode + NoMigration bool +} + +// DefaultOptions returns a StoreOptions populated with default values. +func DefaultOptions() *StoreOptions { + return &StoreOptions{ + NoMigration: false, + } +} + +// OptionModifier is a function signature for modifying the default +// StoreOptions. +type OptionModifier func(*StoreOptions) + +// WithNoMigration allows the database to be opened in read only mode by +// disabling migrations. +func WithNoMigration(b bool) OptionModifier { + return func(o *StoreOptions) { + o.NoMigration = b + } +} diff --git a/payments/db/migration1/payment.go b/payments/db/migration1/payment.go new file mode 100644 index 000000000..53a3d814e --- /dev/null +++ b/payments/db/migration1/payment.go @@ -0,0 +1,689 @@ +package migration1 + +import ( + "errors" + "fmt" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + sphinx "github.com/lightningnetwork/lightning-onion" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" +) + +// FailureReason encodes the reason a payment ultimately failed. +type FailureReason byte + +const ( + // FailureReasonTimeout indicates that the payment did timeout before a + // successful payment attempt was made. + FailureReasonTimeout FailureReason = 0 + + // FailureReasonNoRoute indicates no successful route to the + // destination was found during path finding. + FailureReasonNoRoute FailureReason = 1 + + // FailureReasonError indicates that an unexpected error happened during + // payment. + FailureReasonError FailureReason = 2 + + // FailureReasonPaymentDetails indicates that either the hash is unknown + // or the final cltv delta or amount is incorrect. + FailureReasonPaymentDetails FailureReason = 3 + + // FailureReasonInsufficientBalance indicates that we didn't have enough + // balance to complete the payment. + FailureReasonInsufficientBalance FailureReason = 4 + + // FailureReasonCanceled indicates that the payment was canceled by the + // user. + FailureReasonCanceled FailureReason = 5 + + // TODO(joostjager): Add failure reasons for: + // LocalLiquidityInsufficient, RemoteCapacityInsufficient. +) + +// Error returns a human-readable error string for the FailureReason. +func (r FailureReason) Error() string { + return r.String() +} + +// String returns a human-readable FailureReason. +func (r FailureReason) String() string { + switch r { + case FailureReasonTimeout: + return "timeout" + case FailureReasonNoRoute: + return "no_route" + case FailureReasonError: + return "error" + case FailureReasonPaymentDetails: + return "incorrect_payment_details" + case FailureReasonInsufficientBalance: + return "insufficient_balance" + case FailureReasonCanceled: + return "canceled" + } + + return "unknown" +} + +// PaymentCreationInfo is the information necessary to have ready when +// initiating a payment, moving it into state InFlight. +type PaymentCreationInfo struct { + // PaymentIdentifier is the hash this payment is paying to in case of + // non-AMP payments, and the SetID for AMP payments. + PaymentIdentifier lntypes.Hash + + // Value is the amount we are paying. + Value lnwire.MilliSatoshi + + // CreationTime is the time when this payment was initiated. + CreationTime time.Time + + // PaymentRequest is the full payment request, if any. + PaymentRequest []byte + + // FirstHopCustomRecords are the TLV records that are to be sent to the + // first hop of this payment. These records will be transmitted via the + // wire message (UpdateAddHTLC) only and therefore do not affect the + // onion payload size. + FirstHopCustomRecords lnwire.CustomRecords +} + +// String returns a human-readable description of the payment creation info. +func (p *PaymentCreationInfo) String() string { + return fmt.Sprintf("payment_id=%v, amount=%v, created_at=%v", + p.PaymentIdentifier, p.Value, p.CreationTime) +} + +// HTLCAttemptInfo contains static information about a specific HTLC attempt +// for a payment. This information is used by the router to handle any errors +// coming back after an attempt is made, and to query the switch about the +// status of the attempt. +type HTLCAttemptInfo struct { + // AttemptID is the unique ID used for this attempt. + AttemptID uint64 + + // sessionKey is the raw bytes ephemeral key used for this attempt. + // These bytes are lazily read off disk to save ourselves the expensive + // EC operations used by btcec.PrivKeyFromBytes. + sessionKey [btcec.PrivKeyBytesLen]byte + + // cachedSessionKey is our fully deserialized sesionKey. This value + // may be nil if the attempt has just been read from disk and its + // session key has not been used yet. + cachedSessionKey *btcec.PrivateKey + + // Route is the route attempted to send the HTLC. + Route Route + + // AttemptTime is the time at which this HTLC was attempted. + AttemptTime time.Time + + // Hash is the hash used for this single HTLC attempt. For AMP payments + // this will differ across attempts, for non-AMP payments each attempt + // will use the same hash. This can be nil for older payment attempts, + // in which the payment's PaymentHash in the PaymentCreationInfo should + // be used. + Hash *lntypes.Hash + + // onionBlob is the cached value for onion blob created from the sphinx + // construction. + onionBlob [lnwire.OnionPacketSize]byte + + // circuit is the cached value for sphinx circuit. + circuit *sphinx.Circuit +} + +// SessionKey returns the ephemeral key used for a htlc attempt. This function +// performs expensive ec-ops to obtain the session key if it is not cached. +func (h *HTLCAttemptInfo) SessionKey() *btcec.PrivateKey { + if h.cachedSessionKey == nil { + h.cachedSessionKey, _ = btcec.PrivKeyFromBytes( + h.sessionKey[:], + ) + } + + return h.cachedSessionKey +} + +// HTLCAttempt contains information about a specific HTLC attempt for a given +// payment. It contains the HTLCAttemptInfo used to send the HTLC, as well +// as a timestamp and any known outcome of the attempt. +type HTLCAttempt struct { + HTLCAttemptInfo + + // Settle is the preimage of a successful payment. This serves as a + // proof of payment. It will only be non-nil for settled payments. + // + // NOTE: Can be nil if payment is not settled. + Settle *HTLCSettleInfo + + // Fail is a failure reason code indicating the reason the payment + // failed. It is only non-nil for failed payments. + // + // NOTE: Can be nil if payment is not failed. + Failure *HTLCFailInfo +} + +// HTLCSettleInfo encapsulates the information that augments an HTLCAttempt in +// the event that the HTLC is successful. +type HTLCSettleInfo struct { + // Preimage is the preimage of a successful HTLC. This serves as a proof + // of payment. + Preimage lntypes.Preimage + + // SettleTime is the time at which this HTLC was settled. + SettleTime time.Time +} + +// HTLCFailReason is the reason an htlc failed. +type HTLCFailReason byte + +const ( + // HTLCFailUnknown is recorded for htlcs that failed with an unknown + // reason. + HTLCFailUnknown HTLCFailReason = 0 + + // HTLCFailUnreadable is recorded for htlcs that had a failure message + // that couldn't be decrypted. + HTLCFailUnreadable HTLCFailReason = 1 + + // HTLCFailInternal is recorded for htlcs that failed because of an + // internal error. + HTLCFailInternal HTLCFailReason = 2 + + // HTLCFailMessage is recorded for htlcs that failed with a network + // failure message. + HTLCFailMessage HTLCFailReason = 3 +) + +// HTLCFailInfo encapsulates the information that augments an HTLCAttempt in the +// event that the HTLC fails. +type HTLCFailInfo struct { + // FailTime is the time at which this HTLC was failed. + FailTime time.Time + + // Message is the wire message that failed this HTLC. This field will be + // populated when the failure reason is HTLCFailMessage. + Message lnwire.FailureMessage + + // Reason is the failure reason for this HTLC. + Reason HTLCFailReason + + // The position in the path of the intermediate or final node that + // generated the failure message. Position zero is the sender node. This + // field will be populated when the failure reason is either + // HTLCFailMessage or HTLCFailUnknown. + FailureSourceIndex uint32 +} + +// MPPaymentState wraps a series of info needed for a given payment, which is +// used by both MPP and AMP. This is a memory representation of the payment's +// current state and is updated whenever the payment is read from disk. +type MPPaymentState struct { + // NumAttemptsInFlight specifies the number of HTLCs the payment is + // waiting results for. + NumAttemptsInFlight int + + // RemainingAmt specifies how much more money to be sent. + RemainingAmt lnwire.MilliSatoshi + + // FeesPaid specifies the total fees paid so far that can be used to + // calculate remaining fee budget. + FeesPaid lnwire.MilliSatoshi + + // HasSettledHTLC is true if at least one of the payment's HTLCs is + // settled. + HasSettledHTLC bool + + // PaymentFailed is true if the payment has been marked as failed with + // a reason. + PaymentFailed bool +} + +// MPPayment is a wrapper around a payment's PaymentCreationInfo and +// HTLCAttempts. All payments will have the PaymentCreationInfo set, any +// HTLCs made in attempts to be completed will populated in the HTLCs slice. +// Each populated HTLCAttempt represents an attempted HTLC, each of which may +// have the associated Settle or Fail struct populated if the HTLC is no longer +// in-flight. +type MPPayment struct { + // SequenceNum is a unique identifier used to sort the payments in + // order of creation. + SequenceNum uint64 + + // Info holds all static information about this payment, and is + // populated when the payment is initiated. + Info *PaymentCreationInfo + + // HTLCs holds the information about individual HTLCs that we send in + // order to make the payment. + HTLCs []HTLCAttempt + + // FailureReason is the failure reason code indicating the reason the + // payment failed. + // + // NOTE: Will only be set once the daemon has given up on the payment + // altogether. + FailureReason *FailureReason + + // Status is the current PaymentStatus of this payment. + Status PaymentStatus + + // State is the current state of the payment that holds a number of key + // insights and is used to determine what to do on each payment loop + // iteration. + State *MPPaymentState +} + +// Terminated returns a bool to specify whether the payment is in a terminal +// state. +func (m *MPPayment) Terminated() bool { + // If the payment is in terminal state, it cannot be updated. + return m.Status.updatable() != nil +} + +// TerminalInfo returns any HTLC settle info recorded. If no settle info is +// recorded, any payment level failure will be returned. If neither a settle +// nor a failure is recorded, both return values will be nil. +func (m *MPPayment) TerminalInfo() (*HTLCAttempt, *FailureReason) { + for _, h := range m.HTLCs { + if h.Settle != nil { + return &h, nil + } + } + + return nil, m.FailureReason +} + +// SentAmt returns the sum of sent amount and fees for HTLCs that are either +// settled or still in flight. +func (m *MPPayment) SentAmt() (lnwire.MilliSatoshi, lnwire.MilliSatoshi) { + var sent, fees lnwire.MilliSatoshi + for _, h := range m.HTLCs { + if h.Failure != nil { + continue + } + + // The attempt was not failed, meaning the amount was + // potentially sent to the receiver. + sent += h.Route.ReceiverAmt() + fees += h.Route.TotalFees() + } + + return sent, fees +} + +// InFlightHTLCs returns the HTLCs that are still in-flight, meaning they have +// not been settled or failed. +func (m *MPPayment) InFlightHTLCs() []HTLCAttempt { + var inflights []HTLCAttempt + for _, h := range m.HTLCs { + if h.Settle != nil || h.Failure != nil { + continue + } + + inflights = append(inflights, h) + } + + return inflights +} + +// GetAttempt returns the specified htlc attempt on the payment. +func (m *MPPayment) GetAttempt(id uint64) (*HTLCAttempt, error) { + // TODO(yy): iteration can be slow, make it into a tree or use BS. + for _, htlc := range m.HTLCs { + htlc := htlc + if htlc.AttemptID == id { + return &htlc, nil + } + } + + return nil, errors.New("htlc attempt not found on payment") +} + +// Registrable returns an error to specify whether adding more HTLCs to the +// payment with its current status is allowed. A payment can accept new HTLC +// registrations when it's newly created, or none of its HTLCs is in a terminal +// state. +func (m *MPPayment) Registrable() error { + // If updating the payment is not allowed, we can't register new HTLCs. + // Otherwise, the status must be either `StatusInitiated` or + // `StatusInFlight`. + if err := m.Status.updatable(); err != nil { + return err + } + + // Exit early if this is not inflight. + if m.Status != StatusInFlight { + return nil + } + + // There are still inflight HTLCs and we need to check whether there + // are settled HTLCs or the payment is failed. If we already have + // settled HTLCs, we won't allow adding more HTLCs. + if m.State.HasSettledHTLC { + return ErrPaymentPendingSettled + } + + // If the payment is already failed, we won't allow adding more HTLCs. + if m.State.PaymentFailed { + return ErrPaymentPendingFailed + } + + // Otherwise we can add more HTLCs. + return nil +} + +// setState creates and attaches a new MPPaymentState to the payment. It also +// updates the payment's status based on its current state. +func (m *MPPayment) setState() error { + // Fetch the total amount and fees that has already been sent in + // settled and still in-flight shards. + sentAmt, fees := m.SentAmt() + + // Sanity check we haven't sent a value larger than the payment amount. + totalAmt := m.Info.Value + if sentAmt > totalAmt { + return fmt.Errorf("%w: sent=%v, total=%v", + ErrSentExceedsTotal, sentAmt, totalAmt) + } + + // Get any terminal info for this payment. + settle, failure := m.TerminalInfo() + + // Now determine the payment's status. + status, err := decidePaymentStatus(m.HTLCs, m.FailureReason) + if err != nil { + return err + } + + // Update the payment state and status. + m.State = &MPPaymentState{ + NumAttemptsInFlight: len(m.InFlightHTLCs()), + RemainingAmt: totalAmt - sentAmt, + FeesPaid: fees, + HasSettledHTLC: settle != nil, + PaymentFailed: failure != nil, + } + m.Status = status + + return nil +} + +// SetState calls the internal method setState. This is a temporary method +// to be used by the tests in routing. Once the tests are updated to use mocks, +// this method can be removed. +// +// TODO(yy): delete. +func (m *MPPayment) SetState() error { + return m.setState() +} + +// NeedWaitAttempts decides whether we need to hold creating more HTLC attempts +// and wait for the results of the payment's inflight HTLCs. Return an error if +// the payment is in an unexpected state. +func (m *MPPayment) NeedWaitAttempts() (bool, error) { + // Check when the remainingAmt is not zero, which means we have more + // money to be sent. + if m.State.RemainingAmt != 0 { + switch m.Status { + // If the payment is newly created, no need to wait for HTLC + // results. + case StatusInitiated: + return false, nil + + // If we have inflight HTLCs, we'll check if we have terminal + // states to decide if we need to wait. + case StatusInFlight: + // We still have money to send, and one of the HTLCs is + // settled. We'd stop sending money and wait for all + // inflight HTLC attempts to finish. + if m.State.HasSettledHTLC { + log.Warnf("payment=%v has remaining amount "+ + "%v, yet at least one of its HTLCs is "+ + "settled", m.Info.PaymentIdentifier, + m.State.RemainingAmt) + + return true, nil + } + + // The payment has a failure reason though we still + // have money to send, we'd stop sending money and wait + // for all inflight HTLC attempts to finish. + if m.State.PaymentFailed { + return true, nil + } + + // Otherwise we don't need to wait for inflight HTLCs + // since we still have money to be sent. + return false, nil + + // We need to send more money, yet the payment is already + // succeeded. Return an error in this case as the receiver is + // violating the protocol. + case StatusSucceeded: + return false, fmt.Errorf("%w: parts of the payment "+ + "already succeeded but still have remaining "+ + "amount %v", ErrPaymentInternal, + m.State.RemainingAmt) + + // The payment is failed and we have no inflight HTLCs, no need + // to wait. + case StatusFailed: + return false, nil + + // Unknown payment status. + default: + return false, fmt.Errorf("%w: %s", + ErrUnknownPaymentStatus, m.Status) + } + } + + // Now we determine whether we need to wait when the remainingAmt is + // already zero. + switch m.Status { + // When the payment is newly created, yet the payment has no remaining + // amount, return an error. + case StatusInitiated: + return false, fmt.Errorf("%w: %v", + ErrPaymentInternal, m.Status) + + // If the payment is inflight, we must wait. + // + // NOTE: an edge case is when all HTLCs are failed while the payment is + // not failed we'd still be in this inflight state. However, since the + // remainingAmt is zero here, it means we cannot be in that state as + // otherwise the remainingAmt would not be zero. + case StatusInFlight: + return true, nil + + // If the payment is already succeeded, no need to wait. + case StatusSucceeded: + return false, nil + + // If the payment is already failed, yet the remaining amount is zero, + // return an error as this indicates an error state. We will only each + // this status when there are no inflight HTLCs and the payment is + // marked as failed with a reason, which means the remainingAmt must + // not be zero because our sentAmt is zero. + case StatusFailed: + return false, fmt.Errorf("%w: %v", + ErrPaymentInternal, m.Status) + + // Unknown payment status. + default: + return false, fmt.Errorf("%w: %s", + ErrUnknownPaymentStatus, m.Status) + } +} + +// GetState returns the internal state of the payment. +func (m *MPPayment) GetState() *MPPaymentState { + return m.State +} + +// GetStatus returns the current status of the payment. +func (m *MPPayment) GetStatus() PaymentStatus { + return m.Status +} + +// GetHTLCs returns all the HTLCs for this payment. +func (m *MPPayment) GetHTLCs() []HTLCAttempt { + return m.HTLCs +} + +// AllowMoreAttempts is used to decide whether we can safely attempt more HTLCs +// for a given payment state. Return an error if the payment is in an +// unexpected state. +func (m *MPPayment) AllowMoreAttempts() (bool, error) { + // Now check whether the remainingAmt is zero or not. If we don't have + // any remainingAmt, no more HTLCs should be made. + if m.State.RemainingAmt == 0 { + // If the payment is newly created, yet we don't have any + // remainingAmt, return an error. + if m.Status == StatusInitiated { + return false, fmt.Errorf("%w: initiated payment has "+ + "zero remainingAmt", + ErrPaymentInternal) + } + + // Otherwise, exit early since all other statuses with zero + // remainingAmt indicate no more HTLCs can be made. + return false, nil + } + + // Otherwise, the remaining amount is not zero, we now decide whether + // to make more attempts based on the payment's current status. + // + // If at least one of the payment's attempts is settled, yet we haven't + // sent all the amount, it indicates something is wrong with the peer + // as the preimage is received. In this case, return an error state. + if m.Status == StatusSucceeded { + return false, fmt.Errorf("%w: payment already succeeded but "+ + "still have remaining amount %v", + ErrPaymentInternal, m.State.RemainingAmt) + } + + // Now check if we can register a new HTLC. + err := m.Registrable() + if err != nil { + log.Warnf("Payment(%v): cannot register HTLC attempt: %v, "+ + "current status: %s", m.Info.PaymentIdentifier, + err, m.Status) + + return false, nil + } + + // Now we know we can register new HTLCs. + return true, nil +} + +// verifyAttempt validates that a new HTLC attempt is compatible with the +// existing payment and its in-flight HTLCs. This function checks: +// 1. MPP (Multi-Path Payment) compatibility between attempts +// 2. Blinded payment consistency +// 3. Amount validation +// 4. Total payment amount limits +func verifyAttempt(payment *MPPayment, attempt *HTLCAttemptInfo) error { + // If the final hop has encrypted data, then we know this is a + // blinded payment. In blinded payments, MPP records are not set + // for split payments and the recipient is responsible for using + // a consistent PathID across the various encrypted data + // payloads that we received from them for this payment. All we + // need to check is that the total amount field for each HTLC + // in the split payment is correct. + isBlinded := len(attempt.Route.FinalHop().EncryptedData) != 0 + + // For blinded payments, the last hop must set the total amount. + if isBlinded { + if attempt.Route.FinalHop().TotalAmtMsat == 0 { + return ErrBlindedPaymentMissingTotalAmount + } + } + + // Make sure any existing shards match the new one with regards + // to MPP options. + mpp := attempt.Route.FinalHop().MPP + + // MPP records should not be set for attempts to blinded paths. + if isBlinded && mpp != nil { + return ErrMPPRecordInBlindedPayment + } + + for _, h := range payment.InFlightHTLCs() { + hMpp := h.Route.FinalHop().MPP + hBlinded := len(h.Route.FinalHop().EncryptedData) != 0 + + // If this is a blinded payment, then no existing HTLCs + // should have MPP records. + if isBlinded && hMpp != nil { + return ErrMPPRecordInBlindedPayment + } + + // If the payment is blinded (previous attempts used blinded + // paths) and the attempt is not, or vice versa, return an + // error. + if isBlinded != hBlinded { + return ErrMixedBlindedAndNonBlindedPayments + } + + // If this is a blinded payment, then we just need to + // check that the TotalAmtMsat field for this shard + // is equal to that of any other shard in the same + // payment. + if isBlinded { + if attempt.Route.FinalHop().TotalAmtMsat != + h.Route.FinalHop().TotalAmtMsat { + + return ErrBlindedPaymentTotalAmountMismatch + } + + continue + } + + switch { + // We tried to register a non-MPP attempt for a MPP + // payment. + case mpp == nil && hMpp != nil: + return ErrMPPayment + + // We tried to register a MPP shard for a non-MPP + // payment. + case mpp != nil && hMpp == nil: + return ErrNonMPPayment + + // Non-MPP payment, nothing more to validate. + case mpp == nil: + continue + } + + // Check that MPP options match. + if mpp.PaymentAddr() != hMpp.PaymentAddr() { + return ErrMPPPaymentAddrMismatch + } + + if mpp.TotalMsat() != hMpp.TotalMsat() { + return ErrMPPTotalAmountMismatch + } + } + + // If this is a non-MPP attempt, it must match the total amount + // exactly. Note that a blinded payment is considered an MPP + // attempt. + amt := attempt.Route.ReceiverAmt() + if !isBlinded && mpp == nil && amt != payment.Info.Value { + return ErrValueMismatch + } + + // Ensure we aren't sending more than the total payment amount. + sentAmt, _ := payment.SentAmt() + if sentAmt+amt > payment.Info.Value { + return fmt.Errorf("%w: attempted=%v, payment amount=%v", + ErrValueExceedsAmt, sentAmt+amt, payment.Info.Value) + } + + return nil +} diff --git a/payments/db/migration1/payment_status.go b/payments/db/migration1/payment_status.go new file mode 100644 index 000000000..16c4b90fb --- /dev/null +++ b/payments/db/migration1/payment_status.go @@ -0,0 +1,257 @@ +package migration1 + +import ( + "fmt" +) + +// PaymentStatus represent current status of payment. +type PaymentStatus byte + +const ( + // NOTE: PaymentStatus = 0 was previously used for status unknown and + // is now deprecated. + + // StatusInitiated is the status where a payment has just been + // initiated. + StatusInitiated PaymentStatus = 1 + + // StatusInFlight is the status where a payment has been initiated, but + // a response has not been received. + StatusInFlight PaymentStatus = 2 + + // StatusSucceeded is the status where a payment has been initiated and + // the payment was completed successfully. + StatusSucceeded PaymentStatus = 3 + + // StatusFailed is the status where a payment has been initiated and a + // failure result has come back. + StatusFailed PaymentStatus = 4 +) + +// errPaymentStatusUnknown is returned when a payment has an unknown status. +var errPaymentStatusUnknown = fmt.Errorf("unknown payment status") + +// String returns readable representation of payment status. +func (ps PaymentStatus) String() string { + switch ps { + case StatusInitiated: + return "Initiated" + + case StatusInFlight: + return "In Flight" + + case StatusSucceeded: + return "Succeeded" + + case StatusFailed: + return "Failed" + + default: + return "Unknown" + } +} + +// initializable returns an error to specify whether initiating the payment +// with its current status is allowed. A payment can only be initialized if it +// hasn't been created yet or already failed. +func (ps PaymentStatus) initializable() error { + switch ps { + // The payment has been created already. We will disallow creating it + // again in case other goroutines have already been creating HTLCs for + // it. + case StatusInitiated: + return ErrPaymentExists + + // We already have an InFlight payment on the network. We will disallow + // any new payments. + case StatusInFlight: + return ErrPaymentInFlight + + // The payment has been attempted and is succeeded so we won't allow + // creating it again. + case StatusSucceeded: + return ErrAlreadyPaid + + // We allow retrying failed payments. + case StatusFailed: + return nil + + default: + return fmt.Errorf("%w: %v", ErrUnknownPaymentStatus, + ps) + } +} + +// removable returns an error to specify whether deleting the payment with its +// current status is allowed. A payment cannot be safely deleted if it has +// inflight HTLCs. +func (ps PaymentStatus) removable() error { + switch ps { + // The payment has been created but has no HTLCs and can be removed. + case StatusInitiated: + return nil + + // There are still inflight HTLCs and the payment needs to wait for the + // final outcomes. + case StatusInFlight: + return ErrPaymentInFlight + + // The payment has been attempted and is succeeded and is allowed to be + // removed. + case StatusSucceeded: + return nil + + // Failed payments are allowed to be removed. + case StatusFailed: + return nil + + default: + return fmt.Errorf("%w: %v", ErrUnknownPaymentStatus, + ps) + } +} + +// updatable returns an error to specify whether the payment's HTLCs can be +// updated. A payment can update its HTLCs when it has inflight HTLCs. +func (ps PaymentStatus) updatable() error { + switch ps { + // Newly created payments can be updated. + case StatusInitiated: + return nil + + // Inflight payments can be updated. + case StatusInFlight: + return nil + + // If the payment has a terminal condition, we won't allow any updates. + case StatusSucceeded: + return ErrPaymentAlreadySucceeded + + case StatusFailed: + return ErrPaymentAlreadyFailed + + default: + return fmt.Errorf("%w: %v", ErrUnknownPaymentStatus, + ps) + } +} + +// decidePaymentStatus uses the payment's DB state to determine a memory status +// that's used by the payment router to decide following actions. +// Together, we use four variables to determine the payment's status, +// - inflight: whether there are any pending HTLCs. +// - settled: whether any of the HTLCs has been settled. +// - htlc failed: whether any of the HTLCs has been failed. +// - payment failed: whether the payment has been marked as failed. +// +// Based on the above variables, we derive the status using the following +// table, +// | inflight | settled | htlc failed | payment failed | status | +// |:--------:|:-------:|:-----------:|:--------------:|:--------------------:| +// | true | true | true | true | StatusInFlight | +// | true | true | true | false | StatusInFlight | +// | true | true | false | true | StatusInFlight | +// | true | true | false | false | StatusInFlight | +// | true | false | true | true | StatusInFlight | +// | true | false | true | false | StatusInFlight | +// | true | false | false | true | StatusInFlight | +// | true | false | false | false | StatusInFlight | +// | false | true | true | true | StatusSucceeded | +// | false | true | true | false | StatusSucceeded | +// | false | true | false | true | StatusSucceeded | +// | false | true | false | false | StatusSucceeded | +// | false | false | true | true | StatusFailed | +// | false | false | true | false | StatusInFlight | +// | false | false | false | true | StatusFailed | +// | false | false | false | false | StatusInitiated | +// +// When `inflight`, `settled`, `htlc failed`, and `payment failed` are false, +// this indicates the payment is newly created and hasn't made any HTLCs yet. +// When `inflight` and `settled` are false, `htlc failed` is true yet `payment +// failed` is false, this indicates all the payment's HTLCs have occurred a +// temporarily failure and the payment is still in-flight. +func decidePaymentStatus(htlcs []HTLCAttempt, + reason *FailureReason) (PaymentStatus, error) { + + var ( + inflight bool + htlcSettled bool + htlcFailed bool + paymentFailed bool + ) + + // If we have a failure reason, the payment is failed. + if reason != nil { + paymentFailed = true + } + + // Go through all HTLCs for this payment, check whether we have any + // settled HTLC, and any still in-flight. + for _, h := range htlcs { + if h.Failure != nil { + htlcFailed = true + continue + } + + if h.Settle != nil { + htlcSettled = true + continue + } + + // If any of the HTLCs are not failed nor settled, we + // still have inflight HTLCs. + inflight = true + } + + // Use the DB state to determine the status of the payment. + switch { + // If we have inflight HTLCs, no matter we have settled or failed + // HTLCs, or the payment failed, we still consider it inflight so we + // inform upper systems to wait for the results. + case inflight: + return StatusInFlight, nil + + // If we have no in-flight HTLCs, and at least one of the HTLCs is + // settled, the payment succeeded. + // + // NOTE: when reaching this case, paymentFailed could be true, which + // means we have a conflicting state for this payment. We choose to + // mark the payment as succeeded because it's the receiver's + // responsibility to only settle the payment iff all HTLCs are + // received. + case htlcSettled: + return StatusSucceeded, nil + + // If we have no in-flight HTLCs, and the payment failure is set, the + // payment is considered failed. + // + // NOTE: when reaching this case, settled must be false. + case paymentFailed: + return StatusFailed, nil + + // If we have no in-flight HTLCs, yet the payment is NOT failed, it + // means all the HTLCs are failed. In this case we can attempt more + // HTLCs. + // + // NOTE: when reaching this case, both settled and paymentFailed must + // be false. + case htlcFailed: + return StatusInFlight, nil + + // If none of the HTLCs is either settled or failed, and we have no + // inflight HTLCs, this means the payment has no HTLCs created yet. + // + // NOTE: when reaching this case, both settled and paymentFailed must + // be false. + case !htlcFailed: + return StatusInitiated, nil + + // Otherwise an impossible state is reached. + // + // NOTE: we should never end up here. + default: + log.Error("Impossible payment state reached") + return 0, fmt.Errorf("%w: payment is corrupted", + errPaymentStatusUnknown) + } +} diff --git a/payments/db/migration1/query.go b/payments/db/migration1/query.go new file mode 100644 index 000000000..1fab2fbd9 --- /dev/null +++ b/payments/db/migration1/query.go @@ -0,0 +1,75 @@ +package migration1 + +const ( + // DefaultMaxPayments is the default maximum number of payments returned + // in the payments query pagination. + DefaultMaxPayments = 100 +) + +// Query represents a query to the payments database starting or ending +// at a certain offset index. The number of retrieved records can be limited. +type Query struct { + // IndexOffset determines the starting point of the payments query and + // is always exclusive. In normal order, the query starts at the next + // higher (available) index compared to IndexOffset. In reversed order, + // the query ends at the next lower (available) index compared to the + // IndexOffset. In the case of a zero index_offset, the query will start + // with the oldest payment when paginating forwards, or will end with + // the most recent payment when paginating backwards. + IndexOffset uint64 + + // MaxPayments is the maximal number of payments returned in the + // payments query. + MaxPayments uint64 + + // Reversed gives a meaning to the IndexOffset. If reversed is set to + // true, the query will fetch payments with indices lower than the + // IndexOffset, otherwise, it will return payments with indices greater + // than the IndexOffset. + Reversed bool + + // If IncludeIncomplete is true, then return payments that have not yet + // fully completed. This means that pending payments, as well as failed + // payments will show up if this field is set to true. + IncludeIncomplete bool + + // CountTotal indicates that all payments currently present in the + // payment index (complete and incomplete) should be counted. + CountTotal bool + + // CreationDateStart, expressed in Unix seconds, if set, filters out + // all payments with a creation date greater than or equal to it. + CreationDateStart int64 + + // CreationDateEnd, expressed in Unix seconds, if set, filters out all + // payments with a creation date less than or equal to it. + CreationDateEnd int64 +} + +// Response contains the result of a query to the payments database. +// It includes the set of payments that match the query and integers which +// represent the index of the first and last item returned in the series of +// payments. These integers allow callers to resume their query in the event +// that the query's response exceeds the max number of returnable events. +type Response struct { + // Payments is the set of payments returned from the database for the + // Query. + Payments []*MPPayment + + // FirstIndexOffset is the index of the first element in the set of + // returned MPPayments. Callers can use this to resume their query + // in the event that the slice has too many events to fit into a single + // response. The offset can be used to continue reverse pagination. + FirstIndexOffset uint64 + + // LastIndexOffset is the index of the last element in the set of + // returned MPPayments. Callers can use this to resume their query + // in the event that the slice has too many events to fit into a single + // response. The offset can be used to continue forward pagination. + LastIndexOffset uint64 + + // TotalCount represents the total number of payments that are currently + // stored in the payment database. This will only be set if the + // CountTotal field in the query was set to true. + TotalCount uint64 +} diff --git a/payments/db/migration1/route.go b/payments/db/migration1/route.go new file mode 100644 index 000000000..dc9a22162 --- /dev/null +++ b/payments/db/migration1/route.go @@ -0,0 +1,138 @@ +package migration1 + +import ( + "fmt" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/tlv" +) + +// Vertex is a frozen local type representing a 33-byte compressed public key, +// equivalent to route.Vertex at the time this migration was written. +type Vertex [33]byte + +// NewVertex returns a new Vertex from a compressed public key. +func NewVertex(pub *btcec.PublicKey) Vertex { + var v Vertex + copy(v[:], pub.SerializeCompressed()) + + return v +} + +// NewVertexFromBytes returns a new Vertex from a serialized compressed public +// key byte slice. +func NewVertexFromBytes(b []byte) (Vertex, error) { + if len(b) != 33 { + return Vertex{}, fmt.Errorf("invalid vertex length %d, "+ + "expected 33", len(b)) + } + + var v Vertex + copy(v[:], b) + + return v, nil +} + +// Hop is a frozen local snapshot of route.Hop, containing exactly the fields +// that existed at the time this migration was written. This ensures the +// migration's serialization boundary is independent of future changes to +// route.Hop, in particular the planned removal of LegacyPayload once the KV +// backend is phased out. +type Hop struct { + // PubKeyBytes is the raw bytes of the public key of the target node. + PubKeyBytes Vertex + + // ChannelID is the unique channel ID for the channel. + ChannelID uint64 + + // OutgoingTimeLock is the timelock value that should be used when + // crafting the _outgoing_ HTLC from this hop. + OutgoingTimeLock uint32 + + // AmtToForward is the amount that this hop will forward to the next + // hop. + AmtToForward lnwire.MilliSatoshi + + // MPP encapsulates the data required for option_mpp. This field should + // only be set for the final hop. + MPP *record.MPP + + // AMP encapsulates the data required for option_amp. This field should + // only be set for the final hop. + AMP *record.AMP + + // CustomRecords if non-nil are a set of additional TLV records that + // should be included in the forwarding instructions for this node. + CustomRecords record.CustomSet + + // LegacyPayload signals that this node doesn't understand the new TLV + // payload, so the legacy payload format must be used. + // + // NOTE: This field is preserved here even though it is marked for + // removal in the live route.Hop, because old KV data may have been + // serialized with LegacyPayload=true and the migration must be able to + // deserialize it correctly. + LegacyPayload bool + + // Metadata is additional data sent along with the payment to the + // payee. + Metadata []byte + + // EncryptedData is an encrypted data blob included for hops that are + // part of a blinded route. + EncryptedData []byte + + // BlindingPoint is an ephemeral public key used by introduction nodes + // in blinded routes. + BlindingPoint *btcec.PublicKey + + // TotalAmtMsat is the total amount for a blinded payment. This field + // should only be set for the final hop in a blinded path. + TotalAmtMsat lnwire.MilliSatoshi +} + +// Route is a frozen local snapshot of route.Route, containing exactly the +// fields that existed at the time this migration was written. +type Route struct { + // TotalTimeLock is the cumulative (final) time lock across the entire + // route. + TotalTimeLock uint32 + + // TotalAmount is the total amount of funds required to complete a + // payment over this route, including fees. + TotalAmount lnwire.MilliSatoshi + + // SourcePubKey is the pubkey of the node where this route originates. + SourcePubKey Vertex + + // Hops contains details concerning the specific forwarding details at + // each hop. + Hops []*Hop + + // FirstHopAmount is the amount that should actually be sent to the + // first hop. Only differs from TotalAmount for custom channels. + FirstHopAmount tlv.RecordT[ + tlv.TlvType0, tlv.BigSizeT[lnwire.MilliSatoshi], + ] + + // FirstHopWireCustomRecords is a set of custom TLV records to include + // in the wire message sent to the first hop. + FirstHopWireCustomRecords lnwire.CustomRecords +} + +// FinalHop returns the last hop in the route. +func (r *Route) FinalHop() *Hop { + return r.Hops[len(r.Hops)-1] +} + +// ReceiverAmt returns the amount forwarded to the final hop. +func (r *Route) ReceiverAmt() lnwire.MilliSatoshi { + return r.FinalHop().AmtToForward +} + +// TotalFees returns the total fees paid along the route. +func (r *Route) TotalFees() lnwire.MilliSatoshi { + return r.TotalAmount - r.ReceiverAmt() +} diff --git a/payments/db/migration1/sql_converters.go b/payments/db/migration1/sql_converters.go new file mode 100644 index 000000000..dbf88b2ea --- /dev/null +++ b/payments/db/migration1/sql_converters.go @@ -0,0 +1,274 @@ +package migration1 + +import ( + "bytes" + "fmt" + "strconv" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/tlv" +) + +// dbPaymentToCreationInfo converts database payment data to the +// PaymentCreationInfo struct. +func dbPaymentToCreationInfo(paymentIdentifier []byte, amountMsat int64, + createdAt time.Time, intentPayload []byte, + firstHopCustomRecords lnwire.CustomRecords) *PaymentCreationInfo { + + // This is the payment hash for non-AMP payments and the SetID for AMP + // payments. + var identifier lntypes.Hash + copy(identifier[:], paymentIdentifier) + + return &PaymentCreationInfo{ + PaymentIdentifier: identifier, + Value: lnwire.MilliSatoshi(amountMsat), + // The creation time is stored in the database as UTC but here + // we convert it to local time. + CreationTime: createdAt.Local(), + PaymentRequest: intentPayload, + FirstHopCustomRecords: firstHopCustomRecords, + } +} + +// dbAttemptToHTLCAttempt converts a database HTLC attempt to an HTLCAttempt. +func dbAttemptToHTLCAttempt(dbAttempt sqlc.FetchHtlcAttemptsForPaymentsRow, + hops []sqlc.FetchHopsForAttemptsRow, + hopCustomRecords map[int64][]sqlc.PaymentHopCustomRecord, + routeCustomRecords []sqlc.PaymentAttemptFirstHopCustomRecord) ( + *HTLCAttempt, error) { + + // Convert route-level first hop custom records to CustomRecords map. + var firstHopWireCustomRecords lnwire.CustomRecords + if len(routeCustomRecords) > 0 { + firstHopWireCustomRecords = make(lnwire.CustomRecords) + for _, record := range routeCustomRecords { + firstHopWireCustomRecords[uint64(record.Key)] = + record.Value + } + } + + // Build the route from the database data. + route, err := dbDataToRoute( + hops, hopCustomRecords, dbAttempt.FirstHopAmountMsat, + dbAttempt.RouteTotalTimeLock, dbAttempt.RouteTotalAmount, + dbAttempt.RouteSourceKey, firstHopWireCustomRecords, + ) + if err != nil { + return nil, fmt.Errorf("failed to convert to route: %w", + err) + } + + hash, err := lntypes.MakeHash(dbAttempt.PaymentHash) + if err != nil { + return nil, fmt.Errorf("failed to parse payment "+ + "hash: %w", err) + } + + // Create the attempt info. + var sessionKey [32]byte + copy(sessionKey[:], dbAttempt.SessionKey) + + info := HTLCAttemptInfo{ + AttemptID: uint64(dbAttempt.AttemptIndex), + sessionKey: sessionKey, + Route: *route, + AttemptTime: dbAttempt.AttemptTime, + Hash: &hash, + } + + attempt := &HTLCAttempt{ + HTLCAttemptInfo: info, + } + + // If there's no resolution type, the attempt is still in-flight. + // Return early without processing settlement or failure info. + if !dbAttempt.ResolutionType.Valid { + return attempt, nil + } + + // Add settlement info if present. + if HTLCAttemptResolutionType(dbAttempt.ResolutionType.Int32) == + HTLCAttemptResolutionSettled { + + var preimage lntypes.Preimage + copy(preimage[:], dbAttempt.SettlePreimage) + + attempt.Settle = &HTLCSettleInfo{ + Preimage: preimage, + SettleTime: dbAttempt.ResolutionTime.Time, + } + } + + // Add failure info if present. + if HTLCAttemptResolutionType(dbAttempt.ResolutionType.Int32) == + HTLCAttemptResolutionFailed { + + failure := &HTLCFailInfo{ + FailTime: dbAttempt.ResolutionTime.Time, + } + + if dbAttempt.HtlcFailReason.Valid { + failure.Reason = HTLCFailReason( + dbAttempt.HtlcFailReason.Int32, + ) + } + + if dbAttempt.FailureSourceIndex.Valid { + failure.FailureSourceIndex = uint32( + dbAttempt.FailureSourceIndex.Int32, + ) + } + + // Decode the failure message if present. + if len(dbAttempt.FailureMsg) > 0 { + msg, err := lnwire.DecodeFailureMessage( + bytes.NewReader(dbAttempt.FailureMsg), 0, + ) + if err != nil { + return nil, fmt.Errorf("failed to decode "+ + "failure message: %w", err) + } + failure.Message = msg + } + + attempt.Failure = failure + } + + return attempt, nil +} + +// dbDataToRoute converts database route data to a Route. +func dbDataToRoute(hops []sqlc.FetchHopsForAttemptsRow, + hopCustomRecords map[int64][]sqlc.PaymentHopCustomRecord, + firstHopAmountMsat int64, totalTimeLock int32, totalAmount int64, + sourceKey []byte, firstHopWireCustomRecords lnwire.CustomRecords) ( + *Route, error) { + + if len(hops) == 0 { + return nil, fmt.Errorf("no hops provided") + } + + // Hops are already sorted by hop_index from the SQL query. + routeHops := make([]*Hop, len(hops)) + + for i, hop := range hops { + pubKey, err := NewVertexFromBytes(hop.PubKey) + if err != nil { + return nil, fmt.Errorf("failed to parse pub key: %w", + err) + } + + var channelID uint64 + if hop.Scid != "" { + // The SCID is stored as a string representation + // of the uint64. + var err error + channelID, err = strconv.ParseUint(hop.Scid, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse "+ + "scid: %w", err) + } + } + + routeHop := &Hop{ + PubKeyBytes: pubKey, + ChannelID: channelID, + OutgoingTimeLock: uint32(hop.OutgoingTimeLock), + AmtToForward: lnwire.MilliSatoshi(hop.AmtToForward), + } + + // Add MPP record if present. + if len(hop.MppPaymentAddr) > 0 { + var paymentAddr [32]byte + copy(paymentAddr[:], hop.MppPaymentAddr) + routeHop.MPP = record.NewMPP( + lnwire.MilliSatoshi(hop.MppTotalMsat.Int64), + paymentAddr, + ) + } + + // Add AMP record if present. + if len(hop.AmpRootShare) > 0 { + var rootShare [32]byte + copy(rootShare[:], hop.AmpRootShare) + var setID [32]byte + copy(setID[:], hop.AmpSetID) + + routeHop.AMP = record.NewAMP( + rootShare, setID, + uint32(hop.AmpChildIndex.Int32), + ) + } + + // Add blinding point if present (only for introduction node + // in blinded route). + if len(hop.BlindingPoint) > 0 { + pubKey, err := btcec.ParsePubKey(hop.BlindingPoint) + if err != nil { + return nil, fmt.Errorf("failed to parse "+ + "blinding point: %w", err) + } + routeHop.BlindingPoint = pubKey + } + + // Add encrypted data if present (for all blinded hops). + if len(hop.EncryptedData) > 0 { + routeHop.EncryptedData = hop.EncryptedData + } + + // Add total amount if present (only for final hop in blinded + // route). + if hop.BlindedPathTotalAmt.Valid { + routeHop.TotalAmtMsat = lnwire.MilliSatoshi( + hop.BlindedPathTotalAmt.Int64, + ) + } + + // Add hop-level custom records. + if records, ok := hopCustomRecords[hop.ID]; ok { + routeHop.CustomRecords = make( + record.CustomSet, + ) + for _, rec := range records { + routeHop.CustomRecords[uint64(rec.Key)] = + rec.Value + } + } + + // Add metadata if present. + if len(hop.MetaData) > 0 { + routeHop.Metadata = hop.MetaData + } + + routeHops[i] = routeHop + } + + // Parse the source node public key. + var sourceNode Vertex + copy(sourceNode[:], sourceKey) + + route := &Route{ + TotalTimeLock: uint32(totalTimeLock), + TotalAmount: lnwire.MilliSatoshi(totalAmount), + SourcePubKey: sourceNode, + Hops: routeHops, + FirstHopWireCustomRecords: firstHopWireCustomRecords, + } + + // Set the first hop amount if it is set. + if firstHopAmountMsat != 0 { + route.FirstHopAmount = tlv.NewRecordT[tlv.TlvType0]( + tlv.NewBigSizeT(lnwire.MilliSatoshi( + firstHopAmountMsat, + )), + ) + } + + return route, nil +} diff --git a/payments/db/migration1/sql_store.go b/payments/db/migration1/sql_store.go new file mode 100644 index 000000000..0fe4aa1fc --- /dev/null +++ b/payments/db/migration1/sql_store.go @@ -0,0 +1,2019 @@ +package migration1 + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "math" + "strconv" + "time" + + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" +) + +// PaymentIntentType represents the type of payment intent. +type PaymentIntentType int16 + +const ( + // PaymentIntentTypeBolt11 indicates a BOLT11 invoice payment. + PaymentIntentTypeBolt11 PaymentIntentType = 0 +) + +// HTLCAttemptResolutionType represents the type of HTLC attempt resolution. +type HTLCAttemptResolutionType int32 + +const ( + // HTLCAttemptResolutionSettled indicates the HTLC attempt was settled + // successfully with a preimage. + HTLCAttemptResolutionSettled HTLCAttemptResolutionType = 1 + + // HTLCAttemptResolutionFailed indicates the HTLC attempt failed. + HTLCAttemptResolutionFailed HTLCAttemptResolutionType = 2 +) + +// SQLQueries is a subset of the sqlc.Querier interface that can be used to +// execute queries against the SQL payments tables. +// +//nolint:ll,interfacebloat +type SQLQueries interface { + /* + Payment DB read operations. + */ + FilterPayments(ctx context.Context, query sqlc.FilterPaymentsParams) ([]sqlc.FilterPaymentsRow, error) + FilterPaymentsDesc(ctx context.Context, query sqlc.FilterPaymentsDescParams) ([]sqlc.FilterPaymentsDescRow, error) + FetchPayment(ctx context.Context, paymentIdentifier []byte) (sqlc.FetchPaymentRow, error) + FetchPaymentsByIDs(ctx context.Context, paymentIDs []int64) ([]sqlc.FetchPaymentsByIDsRow, error) + + CountPayments(ctx context.Context) (int64, error) + + FetchHtlcAttemptsForPayments(ctx context.Context, paymentIDs []int64) ([]sqlc.FetchHtlcAttemptsForPaymentsRow, error) + FetchHtlcAttemptResolutionsForPayments(ctx context.Context, paymentIDs []int64) ([]sqlc.FetchHtlcAttemptResolutionsForPaymentsRow, error) + FetchAllInflightAttempts(ctx context.Context, arg sqlc.FetchAllInflightAttemptsParams) ([]sqlc.PaymentHtlcAttempt, error) + FetchHopsForAttempts(ctx context.Context, htlcAttemptIndices []int64) ([]sqlc.FetchHopsForAttemptsRow, error) + + FetchPaymentDuplicates(ctx context.Context, paymentID int64) ([]sqlc.PaymentDuplicate, error) + + FetchPaymentLevelFirstHopCustomRecords(ctx context.Context, paymentIDs []int64) ([]sqlc.PaymentFirstHopCustomRecord, error) + FetchRouteLevelFirstHopCustomRecords(ctx context.Context, htlcAttemptIndices []int64) ([]sqlc.PaymentAttemptFirstHopCustomRecord, error) + FetchHopLevelCustomRecords(ctx context.Context, hopIDs []int64) ([]sqlc.PaymentHopCustomRecord, error) + + /* + Payment DB write operations. + */ + InsertPaymentIntent(ctx context.Context, arg sqlc.InsertPaymentIntentParams) (int64, error) + InsertPayment(ctx context.Context, arg sqlc.InsertPaymentParams) (int64, error) + InsertPaymentFirstHopCustomRecord(ctx context.Context, arg sqlc.InsertPaymentFirstHopCustomRecordParams) error + + InsertHtlcAttempt(ctx context.Context, arg sqlc.InsertHtlcAttemptParams) (int64, error) + InsertRouteHop(ctx context.Context, arg sqlc.InsertRouteHopParams) (int64, error) + InsertRouteHopMpp(ctx context.Context, arg sqlc.InsertRouteHopMppParams) error + InsertRouteHopAmp(ctx context.Context, arg sqlc.InsertRouteHopAmpParams) error + InsertRouteHopBlinded(ctx context.Context, arg sqlc.InsertRouteHopBlindedParams) error + + InsertPaymentAttemptFirstHopCustomRecord(ctx context.Context, arg sqlc.InsertPaymentAttemptFirstHopCustomRecordParams) error + InsertPaymentHopCustomRecord(ctx context.Context, arg sqlc.InsertPaymentHopCustomRecordParams) error + + SettleAttempt(ctx context.Context, arg sqlc.SettleAttemptParams) error + FailAttempt(ctx context.Context, arg sqlc.FailAttemptParams) error + + FailPayment(ctx context.Context, arg sqlc.FailPaymentParams) (sql.Result, error) + + DeletePayment(ctx context.Context, paymentID int64) error + + // DeleteFailedAttempts removes all failed HTLCs from the db for a + // given payment. + DeleteFailedAttempts(ctx context.Context, paymentID int64) error + + /* + Migration specific queries. + + These queries are used ONLY for the one-time migration from KV + to SQL. + */ + + // FetchPaymentsByIDsMig is a migration-only batch fetch that returns + // payment data along with HTLC attempt counts for structural + // validation. + FetchPaymentsByIDsMig(ctx context.Context, paymentIDs []int64) ([]sqlc.FetchPaymentsByIDsMigRow, error) + + // InsertPaymentMig is a migration-only variant of InsertPayment that + // allows setting fail_reason when inserting historical payments, since + // for real payments they have not failed at creation time and so no + // failure reason would exist yet. + InsertPaymentMig(ctx context.Context, arg sqlc.InsertPaymentMigParams) (int64, error) + + // InsertPaymentDuplicateMig inserts a duplicate payment record during + // migration. + InsertPaymentDuplicateMig(ctx context.Context, arg sqlc.InsertPaymentDuplicateMigParams) (int64, error) +} + +// BatchedSQLQueries is a version of the SQLQueries that's capable +// of batched database operations. +type BatchedSQLQueries interface { + SQLQueries + sqldb.BatchedTx[SQLQueries] +} + +// SQLStore represents a storage backend. +type SQLStore struct { + cfg *SQLStoreConfig + db BatchedSQLQueries +} + +// A compile-time constraint to ensure SQLStore implements DB. +var _ DB = (*SQLStore)(nil) + +// SQLStoreConfig holds the configuration for the SQLStore. +type SQLStoreConfig struct { + // QueryConfig holds configuration values for SQL queries. + QueryCfg *sqldb.QueryConfig +} + +// NewSQLStore creates a new SQLStore instance given an open +// BatchedSQLPaymentsQueries storage backend. +func NewSQLStore(cfg *SQLStoreConfig, db BatchedSQLQueries, + options ...OptionModifier) (*SQLStore, error) { + + opts := DefaultOptions() + for _, applyOption := range options { + applyOption(opts) + } + + if opts.NoMigration { + return nil, fmt.Errorf("the NoMigration option is not yet " + + "supported for SQL stores") + } + + return &SQLStore{ + cfg: cfg, + db: db, + }, nil +} + +// fetchPaymentWithCompleteData fetches a payment with all its related data +// including attempts, hops, and custom records from the database. +// This is a convenience wrapper around the batch loading functions for single +// payment operations. +func fetchPaymentWithCompleteData(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, + dbPayment sqlc.PaymentAndIntent) (*MPPayment, error) { + + payment := dbPayment.GetPayment() + + // Load batch data for this single payment. + batchData, err := batchLoadPaymentDetailsData( + ctx, cfg, db, []int64{payment.ID}, + ) + if err != nil { + return nil, fmt.Errorf("failed to load batch data: %w", err) + } + + // Build the payment from the batch data. + return buildPaymentFromBatchData(dbPayment, batchData) +} + +// paymentsCompleteData holds the full payment data when batch loading base +// payment data and all the related data for a payment. +type paymentsCompleteData struct { + *paymentsBaseData + *paymentsDetailsData +} + +// batchLoadPayments loads the full payment data for a batch of payment IDs. +func batchLoadPayments(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, paymentIDs []int64) (*paymentsCompleteData, error) { + + baseData, err := batchLoadpaymentsBaseData(ctx, cfg, db, paymentIDs) + if err != nil { + return nil, fmt.Errorf("failed to load payment base data: %w", + err) + } + + batchData, err := batchLoadPaymentDetailsData(ctx, cfg, db, paymentIDs) + if err != nil { + return nil, fmt.Errorf("failed to load payment batch data: %w", + err) + } + + return &paymentsCompleteData{ + paymentsBaseData: baseData, + paymentsDetailsData: batchData, + }, nil +} + +// paymentsBaseData holds the base payment and intent data for a batch of +// payments. +type paymentsBaseData struct { + // paymentsAndIntents maps payment ID to its payment and intent data. + paymentsAndIntents map[int64]sqlc.PaymentAndIntent +} + +// batchLoadpaymentsBaseData loads the base payment and payment intent data for +// a batch of payment IDs. This complements loadPaymentsBatchData which loads +// related data (attempts, hops, custom records) but not the payment table +// and payment intent table data. +func batchLoadpaymentsBaseData(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, + paymentIDs []int64) (*paymentsBaseData, error) { + + baseData := &paymentsBaseData{ + paymentsAndIntents: make(map[int64]sqlc.PaymentAndIntent), + } + + if len(paymentIDs) == 0 { + return baseData, nil + } + + err := sqldb.ExecuteBatchQuery( + ctx, cfg, paymentIDs, + func(id int64) int64 { return id }, + func(ctx context.Context, ids []int64) ( + []sqlc.FetchPaymentsByIDsRow, error) { + + records, err := db.FetchPaymentsByIDs( + ctx, ids, + ) + + return records, err + }, + func(ctx context.Context, + payment sqlc.FetchPaymentsByIDsRow) error { + + baseData.paymentsAndIntents[payment.ID] = payment + + return nil + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch payment base "+ + "data: %w", err) + } + + return baseData, nil +} + +// paymentsRelatedData holds all the batch-loaded data for multiple payments. +// This does not include the base payment and intent data which is fetched +// separately. It includes the additional data like attempts, hops, hop custom +// records, and route custom records. +type paymentsDetailsData struct { + // paymentCustomRecords maps payment ID to its custom records. + paymentCustomRecords map[int64][]sqlc.PaymentFirstHopCustomRecord + + // attempts maps payment ID to its HTLC attempts. + attempts map[int64][]sqlc.FetchHtlcAttemptsForPaymentsRow + + // hopsByAttempt maps attempt index to its hops. + hopsByAttempt map[int64][]sqlc.FetchHopsForAttemptsRow + + // hopCustomRecords maps hop ID to its custom records. + hopCustomRecords map[int64][]sqlc.PaymentHopCustomRecord + + // routeCustomRecords maps attempt index to its route-level custom + // records. + routeCustomRecords map[int64][]sqlc.PaymentAttemptFirstHopCustomRecord +} + +// batchLoadPaymentCustomRecords loads payment-level custom records for a given +// set of payment IDs. It uses a batch query to fetch all custom records for +// the given payment IDs. +func batchLoadPaymentCustomRecords(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, paymentIDs []int64, + batchData *paymentsDetailsData) error { + + return sqldb.ExecuteBatchQuery( + ctx, cfg, paymentIDs, + func(id int64) int64 { return id }, + func(ctx context.Context, ids []int64) ( + []sqlc.PaymentFirstHopCustomRecord, error) { + + //nolint:ll + records, err := db.FetchPaymentLevelFirstHopCustomRecords( + ctx, ids, + ) + + return records, err + }, + func(ctx context.Context, + record sqlc.PaymentFirstHopCustomRecord) error { + + paymentRecords := + batchData.paymentCustomRecords[record.PaymentID] + + batchData.paymentCustomRecords[record.PaymentID] = + append(paymentRecords, record) + + return nil + }, + ) +} + +// batchLoadHtlcAttempts loads HTLC attempts for all payments and returns all +// attempt indices. It uses a batch query to fetch all attempts for the given +// payment IDs. +func batchLoadHtlcAttempts(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, paymentIDs []int64, + batchData *paymentsDetailsData) ([]int64, error) { + + var allAttemptIndices []int64 + + err := sqldb.ExecuteBatchQuery( + ctx, cfg, paymentIDs, + func(id int64) int64 { return id }, + func(ctx context.Context, ids []int64) ( + []sqlc.FetchHtlcAttemptsForPaymentsRow, error) { + + return db.FetchHtlcAttemptsForPayments(ctx, ids) + }, + func(ctx context.Context, + attempt sqlc.FetchHtlcAttemptsForPaymentsRow) error { + + batchData.attempts[attempt.PaymentID] = append( + batchData.attempts[attempt.PaymentID], attempt, + ) + allAttemptIndices = append( + allAttemptIndices, attempt.AttemptIndex, + ) + + return nil + }, + ) + + return allAttemptIndices, err +} + +// batchLoadHopsForAttempts loads hops for all attempts and returns all hop IDs. +// It uses a batch query to fetch all hops for the given attempt indices. +func batchLoadHopsForAttempts(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, attemptIndices []int64, + batchData *paymentsDetailsData) ([]int64, error) { + + var hopIDs []int64 + + err := sqldb.ExecuteBatchQuery( + ctx, cfg, attemptIndices, + func(idx int64) int64 { return idx }, + func(ctx context.Context, indices []int64) ( + []sqlc.FetchHopsForAttemptsRow, error) { + + return db.FetchHopsForAttempts(ctx, indices) + }, + func(ctx context.Context, + hop sqlc.FetchHopsForAttemptsRow) error { + + attemptHops := + batchData.hopsByAttempt[hop.HtlcAttemptIndex] + + batchData.hopsByAttempt[hop.HtlcAttemptIndex] = + append(attemptHops, hop) + + hopIDs = append(hopIDs, hop.ID) + + return nil + }, + ) + + return hopIDs, err +} + +// batchLoadHopCustomRecords loads hop-level custom records for all hops. It +// uses a batch query to fetch all custom records for the given hop IDs. +func batchLoadHopCustomRecords(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, hopIDs []int64, batchData *paymentsDetailsData) error { + + return sqldb.ExecuteBatchQuery( + ctx, cfg, hopIDs, + func(id int64) int64 { return id }, + func(ctx context.Context, ids []int64) ( + []sqlc.PaymentHopCustomRecord, error) { + + return db.FetchHopLevelCustomRecords(ctx, ids) + }, + func(ctx context.Context, + record sqlc.PaymentHopCustomRecord) error { + + // TODO(ziggie): Can we get rid of this? + // This has to be in place otherwise the + // comparison will not match. + if record.Value == nil { + record.Value = []byte{} + } + + batchData.hopCustomRecords[record.HopID] = append( + batchData.hopCustomRecords[record.HopID], + record, + ) + + return nil + }, + ) +} + +// batchLoadRouteCustomRecords loads route-level first hop custom records for +// all attempts. It uses a batch query to fetch all custom records for the given +// attempt indices. +func batchLoadRouteCustomRecords(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, attemptIndices []int64, + batchData *paymentsDetailsData) error { + + return sqldb.ExecuteBatchQuery( + ctx, cfg, attemptIndices, + func(idx int64) int64 { return idx }, + func(ctx context.Context, indices []int64) ( + []sqlc.PaymentAttemptFirstHopCustomRecord, error) { + + return db.FetchRouteLevelFirstHopCustomRecords( + ctx, indices, + ) + }, + func(ctx context.Context, + record sqlc.PaymentAttemptFirstHopCustomRecord) error { + + idx := record.HtlcAttemptIndex + attemptRecords := batchData.routeCustomRecords[idx] + + batchData.routeCustomRecords[idx] = + append(attemptRecords, record) + + return nil + }, + ) +} + +// paymentStatusData holds lightweight resolution data for computing +// payment status efficiently during deletion operations. +type paymentStatusData struct { + // resolutionTypes maps payment ID to a list of resolution types + // for that payment's HTLC attempts. + resolutionTypes map[int64][]sql.NullInt32 +} + +// batchLoadPaymentResolutions loads only HTLC resolution types for multiple +// payments. This is a lightweight alternative to batchLoadPaymentsRelatedData +// that's optimized for operations that only need to determine payment status. +func batchLoadPaymentResolutions(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, paymentIDs []int64) (*paymentStatusData, error) { + + batchStatusData := &paymentStatusData{ + resolutionTypes: make(map[int64][]sql.NullInt32), + } + + if len(paymentIDs) == 0 { + return batchStatusData, nil + } + + // Use a batch query to fetch all resolution types for the given payment + // IDs. + err := sqldb.ExecuteBatchQuery( + ctx, cfg, paymentIDs, + func(id int64) int64 { return id }, + func(ctx context.Context, ids []int64) ( + []sqlc.FetchHtlcAttemptResolutionsForPaymentsRow, + error) { + + return db.FetchHtlcAttemptResolutionsForPayments( + ctx, ids, + ) + }, + //nolint:ll + func(ctx context.Context, + res sqlc.FetchHtlcAttemptResolutionsForPaymentsRow) error { + + // Group resolutions by payment ID. + batchStatusData.resolutionTypes[res.PaymentID] = append( + batchStatusData.resolutionTypes[res.PaymentID], + res.ResolutionType, + ) + + return nil + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch HTLC resolutions: %w", + err) + } + + return batchStatusData, nil +} + +// loadPaymentResolutions is a single-payment wrapper around +// batchLoadPaymentResolutions for convenience and to prevent duplicate queries +// so we reuse the same batch query for all payments. +func loadPaymentResolutions(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, paymentID int64) ([]sql.NullInt32, error) { + + batchData, err := batchLoadPaymentResolutions( + ctx, cfg, db, []int64{paymentID}, + ) + if err != nil { + return nil, err + } + + return batchData.resolutionTypes[paymentID], nil +} + +// computePaymentStatusFromResolutions determines the payment status from +// resolution types and failure reason without building the complete MPPayment +// structure. This is a lightweight version that builds minimal HTLCAttempt +// structures and delegates to decidePaymentStatus for consistency. +func computePaymentStatusFromResolutions(resolutionTypes []sql.NullInt32, + failReason sql.NullInt32) (PaymentStatus, error) { + + // Build minimal HTLCAttempt slice with only resolution info. + htlcs := make([]HTLCAttempt, len(resolutionTypes)) + for i, resType := range resolutionTypes { + if !resType.Valid { + // NULL resolution_type means in-flight (no Settle, no + // Failure). + continue + } + + switch HTLCAttemptResolutionType(resType.Int32) { + case HTLCAttemptResolutionSettled: + // Mark as settled (preimage details not needed for + // status). + htlcs[i].Settle = &HTLCSettleInfo{} + + case HTLCAttemptResolutionFailed: + // Mark as failed (failure details not needed for + // status). + htlcs[i].Failure = &HTLCFailInfo{} + + default: + return 0, fmt.Errorf("unknown resolution type: %v", + resType.Int32) + } + } + + // Convert fail reason to FailureReason pointer. + var failureReason *FailureReason + if failReason.Valid { + reason := FailureReason(failReason.Int32) + failureReason = &reason + } + + // Use the existing status decision logic. + return decidePaymentStatus(htlcs, failureReason) +} + +// batchLoadPaymentDetailsData loads all related data for multiple payments in +// batch. It uses a batch queries to fetch all data for the given payment IDs. +func batchLoadPaymentDetailsData(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, paymentIDs []int64) (*paymentsDetailsData, error) { + + batchData := &paymentsDetailsData{ + paymentCustomRecords: make( + map[int64][]sqlc.PaymentFirstHopCustomRecord, + ), + attempts: make( + map[int64][]sqlc.FetchHtlcAttemptsForPaymentsRow, + ), + hopsByAttempt: make( + map[int64][]sqlc.FetchHopsForAttemptsRow, + ), + hopCustomRecords: make( + map[int64][]sqlc.PaymentHopCustomRecord, + ), + routeCustomRecords: make( + map[int64][]sqlc.PaymentAttemptFirstHopCustomRecord, + ), + } + + if len(paymentIDs) == 0 { + return batchData, nil + } + + // Load payment-level custom records. + err := batchLoadPaymentCustomRecords( + ctx, cfg, db, paymentIDs, batchData, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch payment custom "+ + "records: %w", err) + } + + // Load HTLC attempts and collect attempt indices. + allAttemptIndices, err := batchLoadHtlcAttempts( + ctx, cfg, db, paymentIDs, batchData, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch HTLC attempts: %w", + err) + } + + if len(allAttemptIndices) == 0 { + // No attempts, return early. + return batchData, nil + } + + // Load hops for all attempts and collect hop IDs. + hopIDs, err := batchLoadHopsForAttempts( + ctx, cfg, db, allAttemptIndices, batchData, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch hops for attempts: %w", + err) + } + + // Load hop-level custom records if there are any hops. + if len(hopIDs) > 0 { + err = batchLoadHopCustomRecords(ctx, cfg, db, hopIDs, batchData) + if err != nil { + return nil, fmt.Errorf("failed to fetch hop custom "+ + "records: %w", err) + } + } + + // Load route-level first hop custom records. + err = batchLoadRouteCustomRecords( + ctx, cfg, db, allAttemptIndices, batchData, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch route custom "+ + "records: %w", err) + } + + return batchData, nil +} + +// buildPaymentFromBatchData builds a complete MPPayment from a database payment +// and pre-loaded batch data. +func buildPaymentFromBatchData(dbPayment sqlc.PaymentAndIntent, + batchData *paymentsDetailsData) (*MPPayment, error) { + + // The query will only return BOLT 11 payment intents or intents with + // no intent type set. + paymentIntent := dbPayment.GetPaymentIntent() + paymentRequest := paymentIntent.IntentPayload + + payment := dbPayment.GetPayment() + + // Get payment-level custom records from batch data. + customRecords := batchData.paymentCustomRecords[payment.ID] + + // Convert to the FirstHopCustomRecords map. + var firstHopCustomRecords lnwire.CustomRecords + if len(customRecords) > 0 { + firstHopCustomRecords = make(lnwire.CustomRecords) + for _, record := range customRecords { + firstHopCustomRecords[uint64(record.Key)] = record.Value + } + } + + // Convert database payment data to the PaymentCreationInfo struct. + info := dbPaymentToCreationInfo( + payment.PaymentIdentifier, payment.AmountMsat, + payment.CreatedAt, paymentRequest, firstHopCustomRecords, + ) + + // Get all HTLC attempts from batch data for a given payment. + dbAttempts := batchData.attempts[payment.ID] + + // Convert all attempts to HTLCAttempt structs using the pre-loaded + // batch data. + attempts := make([]HTLCAttempt, 0, len(dbAttempts)) + for _, dbAttempt := range dbAttempts { + attemptIndex := dbAttempt.AttemptIndex + // Convert the batch row type to the single row type. + attempt, err := dbAttemptToHTLCAttempt( + dbAttempt, batchData.hopsByAttempt[attemptIndex], + batchData.hopCustomRecords, + batchData.routeCustomRecords[attemptIndex], + ) + if err != nil { + return nil, fmt.Errorf("failed to convert attempt "+ + "%d: %w", attemptIndex, err) + } + attempts = append(attempts, *attempt) + } + + // Set the failure reason if present. + // + // TODO(ziggie): Rename it to Payment Memo in the database? + var failureReason *FailureReason + if payment.FailReason.Valid { + reason := FailureReason(payment.FailReason.Int32) + failureReason = &reason + } + + mpPayment := &MPPayment{ + SequenceNum: uint64(payment.ID), + Info: info, + HTLCs: attempts, + FailureReason: failureReason, + } + + // The status and state will be determined by calling + // SetState after construction. + if err := mpPayment.SetState(); err != nil { + return nil, fmt.Errorf("failed to set payment state: %w", err) + } + + return mpPayment, nil +} + +// QueryPayments queries and retrieves payments from the database with support +// for filtering, pagination, and efficient batch loading of related data. +// +// The function accepts a Query parameter that controls: +// - Pagination: IndexOffset specifies where to start (exclusive), and +// MaxPayments limits the number of results returned +// - Ordering: Reversed flag determines if results are returned in reverse +// chronological order +// - Filtering: CreationDateStart/End filter by creation time, and +// IncludeIncomplete controls whether non-succeeded payments are included +// - Metadata: CountTotal flag determines if the total payment count should +// be calculated +// +// The function optimizes performance by loading all related data (HTLCs, +// sequences, failure reasons, etc.) for multiple payments in a single batch +// query, rather than fetching each payment's data individually. +// +// Returns a Response containing: +// - Payments: the list of matching payments with complete data +// - FirstIndexOffset/LastIndexOffset: pagination cursors for the first and +// last payment in the result set +// - TotalCount: total number of payments in the database (if CountTotal was +// requested, otherwise 0) +// +// This is part of the DB interface. +func (s *SQLStore) QueryPayments(ctx context.Context, query Query) (Response, + error) { + + if query.MaxPayments == 0 { + return Response{}, fmt.Errorf("max payments must be non-zero") + } + + var ( + allPayments []*MPPayment + totalCount int64 + initialCursor int64 + ) + + extractCursor := func(row sqlc.FilterPaymentsRow) int64 { + return row.Payment.ID + } + + err := s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + // We first count all payments to determine the total count + // if requested. + if query.CountTotal { + totalPayments, err := db.CountPayments(ctx) + if err != nil { + return fmt.Errorf("failed to count "+ + "payments: %w", err) + } + totalCount = totalPayments + } + + // collectFunc extracts the payment ID from each payment row. + collectFunc := func(row sqlc.FilterPaymentsRow) (int64, error) { + return row.Payment.ID, nil + } + + // batchDataFunc loads all related data for a batch of payments. + batchDataFunc := func(ctx context.Context, paymentIDs []int64) ( + *paymentsDetailsData, error) { + + return batchLoadPaymentDetailsData( + ctx, s.cfg.QueryCfg, db, paymentIDs, + ) + } + + // processPayment processes each payment with the batch-loaded + // data. + processPayment := func(ctx context.Context, + dbPayment sqlc.FilterPaymentsRow, + batchData *paymentsDetailsData) error { + + // Build the payment from the pre-loaded batch data. + mpPayment, err := buildPaymentFromBatchData( + dbPayment, batchData, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment "+ + "with complete data: %w", err) + } + + // To keep compatibility with the old API, we only + // return non-succeeded payments if requested. + if mpPayment.Status != StatusSucceeded && + !query.IncludeIncomplete { + + return nil + } + + if uint64(len(allPayments)) >= query.MaxPayments { + return errMaxPaymentsReached + } + + allPayments = append(allPayments, mpPayment) + + return nil + } + + //nolint:ll + convertFilterPaymentsDescRows := func( + rows []sqlc.FilterPaymentsDescRow) []sqlc.FilterPaymentsRow { + + out := make([]sqlc.FilterPaymentsRow, len(rows)) + for i, row := range rows { + out[i] = sqlc.FilterPaymentsRow{ + Payment: row.Payment, + IntentType: row.IntentType, + IntentPayload: row.IntentPayload, + } + } + + return out + } + + queryFunc := func(ctx context.Context, lastID int64, + limit int32) ([]sqlc.FilterPaymentsRow, error) { + + // Default date bounds: epoch start and far + // future. These are always provided so the SQL + // query uses simple comparisons instead of + // COALESCE (which causes type mismatch on + // Postgres) or OR-based optional filters (which + // can prevent index usage). + createdAfter := time.Unix(0, 0).UTC() + if query.CreationDateStart != 0 { + createdAfter = time.Unix( + query.CreationDateStart, 0, + ).UTC() + } + + createdBefore := time.Date( + 9999, 12, 31, 23, 59, 59, 0, time.UTC, + ) + if query.CreationDateEnd != 0 { + createdBefore = time.Unix( + query.CreationDateEnd, 0, + ).UTC() + } + + filterParams := sqlc.FilterPaymentsParams{ + NumLimit: limit, + CreatedAfter: createdAfter, + CreatedBefore: createdBefore, + // For now there only BOLT 11 payment intents + // exist. + IntentType: sqldb.SQLInt16( + PaymentIntentTypeBolt11, + ), + } + + if query.Reversed { + filterParams.IndexOffsetLet = sqldb.SQLInt64( + lastID, + ) + } else { + filterParams.IndexOffsetGet = sqldb.SQLInt64( + lastID, + ) + } + + if query.Reversed { + rows, err := db.FilterPaymentsDesc( + ctx, sqlc.FilterPaymentsDescParams( + filterParams, + ), + ) + if err != nil { + return nil, err + } + + return convertFilterPaymentsDescRows(rows), nil + } + + return db.FilterPayments(ctx, filterParams) + } + + if query.Reversed { + if query.IndexOffset == 0 { + initialCursor = int64(math.MaxInt64) + } else { + initialCursor = int64(query.IndexOffset) + } + } else { + initialCursor = int64(query.IndexOffset) + } + + return sqldb.ExecuteCollectAndBatchWithSharedDataQuery( + ctx, s.cfg.QueryCfg, initialCursor, queryFunc, + extractCursor, collectFunc, batchDataFunc, + processPayment, + ) + }, func() { + allPayments = nil + }) + + // We make sure we don't return an error if we reached the maximum + // number of payments. Which is the pagination limit for the query + // itself. + if err != nil && !errors.Is(err, errMaxPaymentsReached) { + return Response{}, fmt.Errorf("failed to query payments: %w", + err) + } + + // Handle case where no payments were found + if len(allPayments) == 0 { + return Response{ + Payments: allPayments, + FirstIndexOffset: 0, + LastIndexOffset: 0, + TotalCount: uint64(totalCount), + }, nil + } + + // If the query was reversed, we need to reverse the payment list + // to match the kvstore behavior and return payments in forward order. + if query.Reversed { + for i, j := 0, len(allPayments)-1; i < j; i, j = i+1, j-1 { + allPayments[i], allPayments[j] = allPayments[j], + allPayments[i] + } + } + + return Response{ + Payments: allPayments, + FirstIndexOffset: allPayments[0].SequenceNum, + LastIndexOffset: allPayments[len(allPayments)-1].SequenceNum, + TotalCount: uint64(totalCount), + }, nil +} + +// fetchPaymentByHash fetches a payment by its hash from the database. It is a +// convenience wrapper around the FetchPayment method and checks for +// no rows error and returns ErrPaymentNotInitiated if no payment is found. +func fetchPaymentByHash(ctx context.Context, db SQLQueries, + paymentHash lntypes.Hash) (sqlc.FetchPaymentRow, error) { + + dbPayment, err := db.FetchPayment(ctx, paymentHash[:]) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return dbPayment, fmt.Errorf("failed to fetch payment: %w", err) + } + + if errors.Is(err, sql.ErrNoRows) { + return dbPayment, ErrPaymentNotInitiated + } + + return dbPayment, nil +} + +// FetchPayment retrieves a complete payment record from the database by its +// payment hash. The returned MPPayment includes all payment metadata such as +// creation info, payment status, current state, all HTLC attempts (both +// successful and failed), and the failure reason if the payment has been +// marked as failed. +// +// Returns ErrPaymentNotInitiated if no payment with the given hash exists. +// +// This is part of the DB interface. +func (s *SQLStore) FetchPayment(ctx context.Context, + paymentHash lntypes.Hash) (*MPPayment, error) { + + var mpPayment *MPPayment + + err := s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + dbPayment, err := fetchPaymentByHash(ctx, db, paymentHash) + if err != nil { + return err + } + + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + return nil + }, sqldb.NoOpReset) + if err != nil { + return nil, err + } + + return mpPayment, nil +} + +// FetchInFlightPayments retrieves all payments that have HTLC attempts +// currently in flight (not yet settled or failed). These are payments with at +// least one HTLC attempt that has been registered but has no resolution record. +// +// The SQLStore implementation provides a significant performance improvement +// over the KVStore implementation by using targeted SQL queries instead of +// scanning all payments. +// +// This method is part of the PaymentReader interface, which is embedded in the +// DB interface. It's typically called during node startup to resume monitoring +// of pending payments and ensure HTLCs are properly tracked. +// +// TODO(ziggie): Consider changing the interface to use a callback or iterator +// pattern instead of returning all payments at once. This would allow +// processing payments one at a time without holding them all in memory +// simultaneously: +// - Callback: func FetchInFlightPayments(ctx, func(*MPPayment) error) error +// - Iterator: func FetchInFlightPayments(ctx) (PaymentIterator, error) +// +// While inflight payments are typically a small subset, this would improve +// memory efficiency for nodes with unusually high numbers of concurrent +// payments and would better leverage the existing pagination infrastructure. +func (s *SQLStore) FetchInFlightPayments(ctx context.Context) ([]*MPPayment, + error) { + + var mpPayments []*MPPayment + + err := s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + // Track which payment IDs we've already processed across all + // pages to avoid loading the same payment multiple times when + // multiple inflight attempts belong to the same payment. + processedPayments := make(map[int64]*MPPayment) + + extractCursor := func(row sqlc.PaymentHtlcAttempt) int64 { + return row.AttemptIndex + } + + // collectFunc extracts the payment ID from each attempt row. + collectFunc := func(row sqlc.PaymentHtlcAttempt) ( + int64, error) { + + return row.PaymentID, nil + } + + // batchDataFunc loads payment data for a batch of payment IDs, + // but only for IDs we haven't processed yet. + batchDataFunc := func(ctx context.Context, + paymentIDs []int64) (*paymentsCompleteData, error) { + + // Filter out already-processed payment IDs. + uniqueIDs := make([]int64, 0, len(paymentIDs)) + for _, id := range paymentIDs { + _, processed := processedPayments[id] + if !processed { + uniqueIDs = append(uniqueIDs, id) + } + } + + // If uniqueIDs is empty, the batch load will return + // empty batch data. + return batchLoadPayments( + ctx, s.cfg.QueryCfg, db, uniqueIDs, + ) + } + + // processAttempt processes each attempt. We only build and + // store the payment once per unique payment ID. + processAttempt := func(ctx context.Context, + row sqlc.PaymentHtlcAttempt, + batchData *paymentsCompleteData) error { + + // Skip if we've already processed this payment. + _, processed := processedPayments[row.PaymentID] + if processed { + return nil + } + + dbPayment := batchData.paymentsAndIntents[row.PaymentID] + + // Build the payment from batch data. + mpPayment, err := buildPaymentFromBatchData( + dbPayment, batchData.paymentsDetailsData, + ) + if err != nil { + return fmt.Errorf("failed to build payment: %w", + err) + } + + // Store in our processed map. + processedPayments[row.PaymentID] = mpPayment + + return nil + } + + queryFunc := func(ctx context.Context, lastAttemptIndex int64, + limit int32) ([]sqlc.PaymentHtlcAttempt, + error) { + + return db.FetchAllInflightAttempts(ctx, + sqlc.FetchAllInflightAttemptsParams{ + AttemptIndex: lastAttemptIndex, + Limit: limit, + }, + ) + } + + err := sqldb.ExecuteCollectAndBatchWithSharedDataQuery( + ctx, s.cfg.QueryCfg, int64(-1), queryFunc, + extractCursor, collectFunc, batchDataFunc, + processAttempt, + ) + if err != nil { + return err + } + + // Convert map to slice. + mpPayments = make([]*MPPayment, 0, len(processedPayments)) + for _, payment := range processedPayments { + mpPayments = append(mpPayments, payment) + } + + return nil + }, func() { + mpPayments = nil + }) + if err != nil { + return nil, fmt.Errorf("failed to fetch inflight "+ + "payments: %w", err) + } + + return mpPayments, nil +} + +// DeleteFailedAttempts removes all failed HTLC attempts from the database for +// the specified payment, while preserving the payment record itself and any +// successful or in-flight attempts. +// +// The method performs the following validations before deletion: +// - StatusInitiated: Can delete failed attempts +// - StatusInFlight: Cannot delete, returns ErrPaymentInFlight (active HTLCs +// still on the network) +// - StatusSucceeded: Can delete failed attempts (payment completed) +// - StatusFailed: Can delete failed attempts (payment permanently failed) +// +// This method is idempotent - calling it multiple times on the same payment +// has no adverse effects. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface. It represents +// the final step (step 5) in the payment lifecycle control flow and should be +// called after a payment reaches a terminal state (succeeded or permanently +// failed) to clean up historical failed attempts. +func (s *SQLStore) DeleteFailedAttempts(ctx context.Context, + paymentHash lntypes.Hash) error { + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + dbPayment, err := fetchPaymentByHash(ctx, db, paymentHash) + if err != nil { + return err + } + + paymentStatus, err := computePaymentStatusFromDB( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + if err := paymentStatus.removable(); err != nil { + return fmt.Errorf("cannot delete failed "+ + "attempts for payment %v: %w", paymentHash, err) + } + + // Then we delete the failed attempts for this payment. + return db.DeleteFailedAttempts(ctx, dbPayment.GetPayment().ID) + }, sqldb.NoOpReset) + if err != nil { + return fmt.Errorf("failed to delete failed attempts for "+ + "payment %v: %w", paymentHash, err) + } + + return nil +} + +// computePaymentStatusFromDB computes the payment status by fetching minimal +// data from the database. This is a lightweight query optimized for SQL that +// doesn't load route data, making it significantly more efficient than +// FetchPayment when only the status is needed. +func computePaymentStatusFromDB(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, dbPayment sqlc.PaymentAndIntent) (PaymentStatus, error) { + + payment := dbPayment.GetPayment() + + // Load the resolution types for the payment. + resolutionTypes, err := loadPaymentResolutions( + ctx, cfg, db, payment.ID, + ) + if err != nil { + return 0, fmt.Errorf("failed to load payment resolutions: %w", + err) + } + + // Use the lightweight status computation. + status, err := computePaymentStatusFromResolutions( + resolutionTypes, payment.FailReason, + ) + if err != nil { + return 0, fmt.Errorf("failed to compute payment status: %w", + err) + } + + return status, nil +} + +// DeletePayment removes a payment or its failed HTLC attempts from the +// database based on the failedAttemptsOnly flag. +// +// If failedAttemptsOnly is true, this method deletes only the failed HTLC +// attempts for the payment while preserving the payment record itself and any +// successful or in-flight attempts. This is useful for cleaning up historical +// failed attempts after a payment reaches a terminal state. +// +// If failedAttemptsOnly is false, this method deletes the entire payment +// record including all payment metadata, payment creation info, all HTLC +// attempts (both failed and successful), and associated data such as payment +// intents and custom records. +// +// Before deletion, this method validates the payment status to ensure it's +// safe to delete: +// - StatusInitiated: Can be deleted (no HTLCs sent yet) +// - StatusInFlight: Cannot be deleted, returns ErrPaymentInFlight (active +// HTLCs on the network) +// - StatusSucceeded: Can be deleted (payment completed successfully) +// - StatusFailed: Can be deleted (payment has failed permanently) +// +// Returns an error if the payment has in-flight HTLCs or if the payment +// doesn't exist. +// +// This method is part of the PaymentWriter interface, which is embedded in +// the DB interface. +func (s *SQLStore) DeletePayment(ctx context.Context, paymentHash lntypes.Hash, + failedHtlcsOnly bool) error { + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + dbPayment, err := fetchPaymentByHash(ctx, db, paymentHash) + if err != nil { + return err + } + + paymentStatus, err := computePaymentStatusFromDB( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + if err := paymentStatus.removable(); err != nil { + return fmt.Errorf("payment %v cannot be deleted: %w", + paymentHash, err) + } + + // If we are only deleting failed HTLCs, we delete them. + if failedHtlcsOnly { + return db.DeleteFailedAttempts( + ctx, dbPayment.GetPayment().ID, + ) + } + + // In case we are not deleting failed HTLCs, we delete the + // payment which will cascade delete all related data. + return db.DeletePayment(ctx, dbPayment.GetPayment().ID) + }, sqldb.NoOpReset) + if err != nil { + return fmt.Errorf("failed to delete failed attempts for "+ + "payment %v: %w", paymentHash, err) + } + + return nil +} + +// InitPayment creates a new payment record in the database with the given +// payment hash and creation info. +// +// Before creating the payment, this method checks if a payment with the same +// hash already exists and validates whether initialization is allowed based on +// the existing payment's status: +// - StatusInitiated: Returns ErrPaymentExists (payment already created, +// HTLCs may be in flight) +// - StatusInFlight: Returns ErrPaymentInFlight (payment currently being +// attempted) +// - StatusSucceeded: Returns ErrAlreadyPaid (payment already succeeded) +// - StatusFailed: Allows retry by deleting the old payment record and +// creating a new one +// +// If no existing payment is found, a new payment record is created with +// StatusInitiated and stored with all associated metadata. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface, representing +// the first step in the payment lifecycle control flow. +func (s *SQLStore) InitPayment(ctx context.Context, paymentHash lntypes.Hash, + paymentCreationInfo *PaymentCreationInfo) error { + + // Create the payment in the database. + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + existingPayment, err := db.FetchPayment(ctx, paymentHash[:]) + switch { + // A payment with this hash already exists. We need to check its + // status to see if we can re-initialize. + case err == nil: + paymentStatus, err := computePaymentStatusFromDB( + ctx, s.cfg.QueryCfg, db, existingPayment, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + // Check if the payment is initializable otherwise + // we'll return early. + if err := paymentStatus.initializable(); err != nil { + return fmt.Errorf("payment is not "+ + "initializable: %w", err) + } + + // If the initializable check above passes, then the + // existing payment has failed. So we delete it and + // all of its previous artifacts. We rely on + // cascading deletes to clean up the rest. + err = db.DeletePayment(ctx, existingPayment.Payment.ID) + if err != nil { + return fmt.Errorf("failed to delete "+ + "payment: %w", err) + } + + // An unexpected error occurred while fetching the payment. + case !errors.Is(err, sql.ErrNoRows): + // Some other error occurred + return fmt.Errorf("failed to check existing "+ + "payment: %w", err) + + // The payment does not yet exist, so we can proceed. + default: + } + + // Insert the payment first to get its ID. + paymentID, err := db.InsertPayment( + ctx, sqlc.InsertPaymentParams{ + AmountMsat: int64( + paymentCreationInfo.Value, + ), + CreatedAt: paymentCreationInfo. + CreationTime.UTC(), + PaymentIdentifier: paymentHash[:], + }, + ) + if err != nil { + return fmt.Errorf("failed to insert payment: %w", err) + } + + // If there's a payment request, insert the payment intent. + if len(paymentCreationInfo.PaymentRequest) > 0 { + _, err = db.InsertPaymentIntent( + ctx, sqlc.InsertPaymentIntentParams{ + PaymentID: paymentID, + IntentType: int16( + PaymentIntentTypeBolt11, + ), + IntentPayload: paymentCreationInfo. + PaymentRequest, + }, + ) + if err != nil { + return fmt.Errorf("failed to insert "+ + "payment intent: %w", err) + } + } + + firstHopCustomRecords := paymentCreationInfo. + FirstHopCustomRecords + + for key, value := range firstHopCustomRecords { + err = db.InsertPaymentFirstHopCustomRecord( + ctx, + sqlc.InsertPaymentFirstHopCustomRecordParams{ + PaymentID: paymentID, + Key: int64(key), + Value: value, + }, + ) + if err != nil { + return fmt.Errorf("failed to insert "+ + "payment first hop custom "+ + "record: %w", err) + } + } + + return nil + }, sqldb.NoOpReset) + if err != nil { + return fmt.Errorf("failed to initialize payment: %w", err) + } + + return nil +} + +// insertRouteHops inserts all route hop data for a given set of hops. +func (s *SQLStore) insertRouteHops(ctx context.Context, db SQLQueries, + hops []*Hop, attemptID uint64) error { + + for i, hop := range hops { + // Insert the basic route hop data and get the generated ID. + hopID, err := db.InsertRouteHop(ctx, sqlc.InsertRouteHopParams{ + HtlcAttemptIndex: int64(attemptID), + HopIndex: int32(i), + PubKey: hop.PubKeyBytes[:], + Scid: strconv.FormatUint( + hop.ChannelID, 10, + ), + OutgoingTimeLock: int32(hop.OutgoingTimeLock), + AmtToForward: int64(hop.AmtToForward), + MetaData: hop.Metadata, + }) + if err != nil { + return fmt.Errorf("failed to insert route hop: %w", err) + } + + // Insert the per-hop custom records. + if len(hop.CustomRecords) > 0 { + for key, value := range hop.CustomRecords { + err = db.InsertPaymentHopCustomRecord( + ctx, + sqlc.InsertPaymentHopCustomRecordParams{ + HopID: hopID, + Key: int64(key), + Value: value, + }) + if err != nil { + return fmt.Errorf("failed to insert "+ + "payment hop custom record: %w", + err) + } + } + } + + // Insert MPP data if present. + if hop.MPP != nil { + paymentAddr := hop.MPP.PaymentAddr() + err = db.InsertRouteHopMpp( + ctx, sqlc.InsertRouteHopMppParams{ + HopID: hopID, + PaymentAddr: paymentAddr[:], + TotalMsat: int64(hop.MPP.TotalMsat()), + }) + if err != nil { + return fmt.Errorf("failed to insert "+ + "route hop MPP: %w", err) + } + } + + // Insert AMP data if present. + if hop.AMP != nil { + rootShare := hop.AMP.RootShare() + setID := hop.AMP.SetID() + err = db.InsertRouteHopAmp( + ctx, sqlc.InsertRouteHopAmpParams{ + HopID: hopID, + RootShare: rootShare[:], + SetID: setID[:], + ChildIndex: int32(hop.AMP.ChildIndex()), + }) + if err != nil { + return fmt.Errorf("failed to insert "+ + "route hop AMP: %w", err) + } + } + + // Insert blinded route data if present. Every hop in the + // blinded path must have an encrypted data record. If the + // encrypted data is not present, we skip the insertion. + if hop.EncryptedData == nil { + continue + } + + // The introduction point has a blinding point set. + var blindingPointBytes []byte + if hop.BlindingPoint != nil { + blindingPointBytes = hop.BlindingPoint. + SerializeCompressed() + } + + // The total amount is only set for the final hop in a + // blinded path. + totalAmtMsat := sql.NullInt64{} + if i == len(hops)-1 { + totalAmtMsat = sql.NullInt64{ + Int64: int64(hop.TotalAmtMsat), + Valid: true, + } + } + + err = db.InsertRouteHopBlinded(ctx, + sqlc.InsertRouteHopBlindedParams{ + HopID: hopID, + EncryptedData: hop.EncryptedData, + BlindingPoint: blindingPointBytes, + BlindedPathTotalAmt: totalAmtMsat, + }, + ) + if err != nil { + return fmt.Errorf("failed to insert "+ + "route hop blinded: %w", err) + } + } + + return nil +} + +// RegisterAttempt atomically records a new HTLC attempt for the specified +// payment. The attempt includes the attempt ID, session key, route information +// (hops, timelocks, amounts), and optional data such as MPP/AMP parameters, +// blinded route data, and custom records. +// +// Returns the updated MPPayment with the new attempt appended to the HTLCs +// slice, and the payment state recalculated. Returns an error if the payment +// doesn't exist or validation fails. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface. It represents +// step 2 in the payment lifecycle control flow, called after InitPayment and +// potentially multiple times for multi-path payments. +func (s *SQLStore) RegisterAttempt(ctx context.Context, + paymentHash lntypes.Hash, attempt *HTLCAttemptInfo) (*MPPayment, + error) { + + var mpPayment *MPPayment + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + // Make sure the payment exists. + dbPayment, err := db.FetchPayment(ctx, paymentHash[:]) + if err != nil { + return err + } + + // We fetch the complete payment to determine if the payment is + // registrable. + // + // TODO(ziggie): We could improve the query here since only + // the last hop data is needed here not the complete payment + // data. + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + if err := mpPayment.Registrable(); err != nil { + return fmt.Errorf("htlc attempt not registrable: %w", + err) + } + + // Verify the attempt is compatible with the existing payment. + if err := verifyAttempt(mpPayment, attempt); err != nil { + return fmt.Errorf("failed to verify attempt: %w", err) + } + + // Register the plain HTLC attempt next. + sessionKey := attempt.SessionKey() + sessionKeyBytes := sessionKey.Serialize() + + _, err = db.InsertHtlcAttempt(ctx, sqlc.InsertHtlcAttemptParams{ + PaymentID: dbPayment.Payment.ID, + AttemptIndex: int64(attempt.AttemptID), + SessionKey: sessionKeyBytes, + AttemptTime: attempt.AttemptTime, + PaymentHash: paymentHash[:], + FirstHopAmountMsat: int64( + attempt.Route.FirstHopAmount.Val.Int(), + ), + RouteTotalTimeLock: int32(attempt.Route.TotalTimeLock), + RouteTotalAmount: int64(attempt.Route.TotalAmount), + RouteSourceKey: attempt.Route.SourcePubKey[:], + }) + if err != nil { + return fmt.Errorf("failed to insert HTLC "+ + "attempt: %w", err) + } + + // Insert the route level first hop custom records. + attemptFirstHopCustomRecords := attempt.Route. + FirstHopWireCustomRecords + + for key, value := range attemptFirstHopCustomRecords { + //nolint:ll + err = db.InsertPaymentAttemptFirstHopCustomRecord( + ctx, + sqlc.InsertPaymentAttemptFirstHopCustomRecordParams{ + HtlcAttemptIndex: int64(attempt.AttemptID), + Key: int64(key), + Value: value, + }, + ) + if err != nil { + return fmt.Errorf("failed to insert "+ + "payment attempt first hop custom "+ + "record: %w", err) + } + } + + // Insert the route hops. + err = s.insertRouteHops( + ctx, db, attempt.Route.Hops, attempt.AttemptID, + ) + if err != nil { + return fmt.Errorf("failed to insert route hops: %w", + err) + } + + // We fetch the HTLC attempts again to recalculate the payment + // state after the attempt is registered. This also makes sure + // we have the right data in case multiple attempts are + // registered concurrently. + // + // NOTE: While the caller is responsible for serializing calls + // to RegisterAttempt per payment hash (see PaymentControl + // interface), we still refetch here to guarantee we return + // consistent, up-to-date data that reflects all changes made + // within this transaction. + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + return nil + }, func() { + mpPayment = nil + }) + if err != nil { + return nil, fmt.Errorf("failed to register attempt: %w", err) + } + + return mpPayment, nil +} + +// SettleAttempt marks the specified HTLC attempt as successfully settled, +// recording the payment preimage and settlement time. The preimage serves as +// cryptographic proof of payment and is atomically saved to the database. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface. It represents +// step 3a in the payment lifecycle control flow (step 3b is FailAttempt), +// called after RegisterAttempt when an HTLC successfully completes. +func (s *SQLStore) SettleAttempt(ctx context.Context, paymentHash lntypes.Hash, + attemptID uint64, settleInfo *HTLCSettleInfo) (*MPPayment, error) { + + var mpPayment *MPPayment + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + dbPayment, err := fetchPaymentByHash(ctx, db, paymentHash) + if err != nil { + return err + } + + paymentStatus, err := computePaymentStatusFromDB( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + if err := paymentStatus.updatable(); err != nil { + return fmt.Errorf("payment is not updatable: %w", err) + } + + err = db.SettleAttempt(ctx, sqlc.SettleAttemptParams{ + AttemptIndex: int64(attemptID), + ResolutionTime: time.Now(), + ResolutionType: int32(HTLCAttemptResolutionSettled), + SettlePreimage: settleInfo.Preimage[:], + }) + if err != nil { + return fmt.Errorf("failed to settle attempt: %w", err) + } + + // Fetch the complete payment after we settled the attempt. + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + return nil + }, func() { + mpPayment = nil + }) + if err != nil { + return nil, fmt.Errorf("failed to settle attempt: %w", err) + } + + return mpPayment, nil +} + +// FailAttempt marks the specified HTLC attempt as failed, recording the +// failure reason, failure time, optional failure message, and the index of the +// node in the route that generated the failure. This information is atomically +// saved to the database for debugging and route optimization purposes. +// +// For single-path payments, failing the only attempt may lead to the payment +// being retried or ultimately failed via the Fail method. For multi-shard +// (MPP/AMP) payments, individual shard failures don't necessarily fail the +// entire payment; additional attempts can be registered until sufficient shards +// succeed or the payment is permanently failed. +// +// Returns the updated MPPayment with the attempt marked as failed and the +// payment state recalculated. The payment status remains StatusInFlight if +// other attempts are still in flight, or may transition based on the overall +// payment state. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface. It represents +// step 3b in the payment lifecycle control flow (step 3a is SettleAttempt), +// called after RegisterAttempt when an HTLC fails. +func (s *SQLStore) FailAttempt(ctx context.Context, paymentHash lntypes.Hash, + attemptID uint64, failInfo *HTLCFailInfo) (*MPPayment, error) { + + var mpPayment *MPPayment + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + // Make sure the payment exists. + dbPayment, err := fetchPaymentByHash(ctx, db, paymentHash) + if err != nil { + return err + } + + paymentStatus, err := computePaymentStatusFromDB( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + // We check if the payment is updatable before failing the + // attempt. + if err := paymentStatus.updatable(); err != nil { + return fmt.Errorf("payment is not updatable: %w", err) + } + + var failureMsg bytes.Buffer + if failInfo.Message != nil { + err := lnwire.EncodeFailureMessage( + &failureMsg, failInfo.Message, 0, + ) + if err != nil { + return fmt.Errorf("failed to encode "+ + "failure message: %w", err) + } + } + + err = db.FailAttempt(ctx, sqlc.FailAttemptParams{ + AttemptIndex: int64(attemptID), + ResolutionTime: time.Now(), + ResolutionType: int32(HTLCAttemptResolutionFailed), + FailureSourceIndex: sqldb.SQLInt32( + failInfo.FailureSourceIndex, + ), + HtlcFailReason: sqldb.SQLInt32(failInfo.Reason), + FailureMsg: failureMsg.Bytes(), + }) + if err != nil { + return fmt.Errorf("failed to fail attempt: %w", err) + } + + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + return nil + }, func() { + mpPayment = nil + }) + if err != nil { + return nil, fmt.Errorf("failed to fail attempt: %w", err) + } + + return mpPayment, nil +} + +// Fail records the ultimate reason why a payment failed. This method stores +// the failure reason for record keeping but does not enforce that all HTLC +// attempts are resolved - HTLCs may still be in flight when this is called. +// +// The payment's actual status transition to StatusFailed is determined by the +// payment state calculation, which considers both the recorded failure reason +// and the current state of all HTLC attempts. The status will transition to +// StatusFailed once all HTLCs are resolved and/or a failure reason is recorded. +// +// NOTE: According to the interface contract, this should only be called when +// all active attempts are already failed. However, the implementation allows +// concurrent calls and does not validate this precondition, enabling the last +// failing attempt to record the failure reason without synchronization. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface. It represents +// step 4 in the payment lifecycle control flow. +func (s *SQLStore) Fail(ctx context.Context, paymentHash lntypes.Hash, + reason FailureReason) (*MPPayment, error) { + + var mpPayment *MPPayment + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + result, err := db.FailPayment(ctx, sqlc.FailPaymentParams{ + PaymentIdentifier: paymentHash[:], + FailReason: sqldb.SQLInt32(reason), + }) + if err != nil { + return err + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return err + } + if rowsAffected == 0 { + return ErrPaymentNotInitiated + } + + payment, err := db.FetchPayment(ctx, paymentHash[:]) + if err != nil { + return fmt.Errorf("failed to fetch payment: %w", err) + } + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, payment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + return nil + }, func() { + mpPayment = nil + }) + if err != nil { + return nil, fmt.Errorf("failed to fail payment: %w", err) + } + + return mpPayment, nil +} + +// DeletePayments performs a batch deletion of payments or their failed HTLC +// attempts from the database based on the specified flags. This is a bulk +// operation that iterates through all payments and selectively deletes based +// on the criteria. +// The behavior is controlled by two flags: +// +// If failedAttemptsOnly is true, only failed HTLC attempts are deleted while +// preserving the payment records and any successful or in-flight attempts. +// The return value is always 0 when deleting attempts only. +// +// If failedAttemptsOnly is false, entire payment records are deleted including +// all associated data (HTLCs, metadata, intents). The return value is the +// number of payments deleted. +// +// The failedOnly flag further filters which payments are processed: +// - failedOnly=true, failedAttemptsOnly=true: Delete failed attempts for +// StatusFailed payments only +// - failedOnly=false, failedAttemptsOnly=true: Delete failed attempts for +// all removable payments +// - failedOnly=true, failedAttemptsOnly=false: Delete entire payment records +// for StatusFailed payments only +// - failedOnly=false, failedAttemptsOnly=false: Delete all removable payment +// records (StatusInitiated, StatusSucceeded, StatusFailed) +// +// Safety checks applied to all operations: +// - Payments with StatusInFlight are always skipped (cannot be safely deleted +// while HTLCs are on the network) +// - The payment status must pass the removable() check +// +// Returns the number of complete payments deleted (0 if only deleting failed +// attempts). This is useful for cleanup operations, administrative maintenance, +// or freeing up database storage. +// +// This method is part of the PaymentWriter interface, which is embedded in +// the DB interface. +// +// TODO(ziggie): batch and use iterator instead, moreover we dont need to fetch +// the complete payment data for each payment, we can just fetch the payment ID +// and the resolution types to decide if the payment is removable. +func (s *SQLStore) DeletePayments(ctx context.Context, failedOnly, + failedHtlcsOnly bool) (int, error) { + + var numPayments int + + extractCursor := func(row sqlc.FilterPaymentsRow) int64 { + return row.Payment.ID + } + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + // collectFunc extracts the payment ID from each payment row. + collectFunc := func(row sqlc.FilterPaymentsRow) (int64, error) { + return row.Payment.ID, nil + } + + // batchDataFunc loads only HTLC resolution types for a batch + // of payments, which is sufficient to determine payment status. + batchDataFunc := func(ctx context.Context, paymentIDs []int64) ( + *paymentStatusData, error) { + + return batchLoadPaymentResolutions( + ctx, s.cfg.QueryCfg, db, paymentIDs, + ) + } + + // processPayment processes each payment with the lightweight + // batch-loaded resolution data. + processPayment := func(ctx context.Context, + dbPayment sqlc.FilterPaymentsRow, + batchData *paymentStatusData) error { + + payment := dbPayment.Payment + + // Compute the payment status from resolution types and + // failure reason without building the complete payment. + resolutionTypes := batchData.resolutionTypes[payment.ID] + status, err := computePaymentStatusFromResolutions( + resolutionTypes, payment.FailReason, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + // Payments which are not final yet cannot be deleted. + // we skip them. + if err := status.removable(); err != nil { + return nil + } + + // If we are only deleting failed payments, we skip + // if the payment is not failed. + if failedOnly && status != StatusFailed { + return nil + } + + // If we are only deleting failed HTLCs, we delete them + // and return early. + if failedHtlcsOnly { + return db.DeleteFailedAttempts( + ctx, payment.ID, + ) + } + + // Otherwise we delete the payment. + err = db.DeletePayment(ctx, payment.ID) + if err != nil { + return fmt.Errorf("failed to delete "+ + "payment: %w", err) + } + + numPayments++ + + return nil + } + + queryFunc := func(ctx context.Context, lastID int64, + limit int32) ([]sqlc.FilterPaymentsRow, error) { + + filterParams := sqlc.FilterPaymentsParams{ + NumLimit: limit, + CreatedAfter: time.Unix(0, 0).UTC(), + CreatedBefore: time.Date( + 9999, 12, 31, 23, 59, 59, + 0, time.UTC, + ), + IndexOffsetGet: sqldb.SQLInt64( + lastID, + ), + } + + return db.FilterPayments(ctx, filterParams) + } + + return sqldb.ExecuteCollectAndBatchWithSharedDataQuery( + ctx, s.cfg.QueryCfg, int64(-1), queryFunc, + extractCursor, collectFunc, batchDataFunc, + processPayment, + ) + }, func() { + numPayments = 0 + }) + if err != nil { + return 0, fmt.Errorf("failed to delete payments "+ + "(failedOnly: %v, failedHtlcsOnly: %v): %w", + failedOnly, failedHtlcsOnly, err) + } + + return numPayments, nil +} diff --git a/payments/db/migration1/sqlc/db.go b/payments/db/migration1/sqlc/db.go new file mode 100644 index 000000000..e4d78283b --- /dev/null +++ b/payments/db/migration1/sqlc/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package sqlc + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/payments/db/migration1/sqlc/db_custom.go b/payments/db/migration1/sqlc/db_custom.go new file mode 100644 index 000000000..625e65adc --- /dev/null +++ b/payments/db/migration1/sqlc/db_custom.go @@ -0,0 +1,123 @@ +package sqlc + +import ( + "fmt" + "strings" +) + +// GetTx returns the underlying DBTX (either *sql.DB or *sql.Tx) used by the +// Queries struct. +func (q *Queries) GetTx() DBTX { + return q.db +} + +// makeQueryParams generates a string of query parameters for a SQL query. It is +// meant to replace the `?` placeholders in a SQL query with numbered parameters +// like `$1`, `$2`, etc. This is required for the sqlc /*SLICE:*/ +// workaround. See scripts/gen_sqlc_docker.sh for more details. +func makeQueryParams(numTotalArgs, numListArgs int) string { + if numListArgs == 0 { + return "" + } + + var b strings.Builder + + // Pre-allocate a rough estimation of the buffer size to avoid + // re-allocations. A parameter like $1000, takes 6 bytes. + b.Grow(numListArgs * 6) + + diff := numTotalArgs - numListArgs + for i := 0; i < numListArgs; i++ { + if i > 0 { + // We don't need to check the error here because the + // WriteString method of strings.Builder always returns + // nil. + _, _ = b.WriteString(",") + } + + // We don't need to check the error here because the + // Write method (called by fmt.Fprintf) of strings.Builder + // always returns nil. + _, _ = fmt.Fprintf(&b, "$%d", i+diff+1) + } + + return b.String() +} + +// PaymentAndIntent is an interface that provides access to a payment and its +// associated payment intent. +type PaymentAndIntent interface { + // GetPayment returns the Payment associated with this interface. + GetPayment() Payment + + // GetPaymentIntent returns the PaymentIntent associated with this + // payment. + GetPaymentIntent() PaymentIntent +} + +// GetPayment returns the Payment associated with this interface. +// +// NOTE: This method is part of the PaymentAndIntent interface. +func (r FilterPaymentsRow) GetPayment() Payment { + return r.Payment +} + +// GetPaymentIntent returns the PaymentIntent associated with this payment. +// If the payment has no intent (IntentType is NULL), this returns a zero-value +// PaymentIntent. +// +// NOTE: This method is part of the PaymentAndIntent interface. +func (r FilterPaymentsRow) GetPaymentIntent() PaymentIntent { + if !r.IntentType.Valid { + return PaymentIntent{} + } + + return PaymentIntent{ + IntentType: r.IntentType.Int16, + IntentPayload: r.IntentPayload, + } +} + +// GetPayment returns the Payment associated with this interface. +// +// NOTE: This method is part of the PaymentAndIntent interface. +func (r FetchPaymentRow) GetPayment() Payment { + return r.Payment +} + +// GetPaymentIntent returns the PaymentIntent associated with this payment. +// If the payment has no intent (IntentType is NULL), this returns a zero-value +// PaymentIntent. +// +// NOTE: This method is part of the PaymentAndIntent interface. +func (r FetchPaymentRow) GetPaymentIntent() PaymentIntent { + if !r.IntentType.Valid { + return PaymentIntent{} + } + + return PaymentIntent{ + IntentType: r.IntentType.Int16, + IntentPayload: r.IntentPayload, + } +} + +func (r FetchPaymentsByIDsRow) GetPayment() Payment { + return Payment{ + ID: r.ID, + AmountMsat: r.AmountMsat, + CreatedAt: r.CreatedAt, + PaymentIdentifier: r.PaymentIdentifier, + FailReason: r.FailReason, + } +} + +func (r FetchPaymentsByIDsRow) GetPaymentIntent() PaymentIntent { + if !r.IntentType.Valid { + return PaymentIntent{} + } + + return PaymentIntent{ + IntentType: r.IntentType.Int16, + IntentPayload: r.IntentPayload, + } +} diff --git a/payments/db/migration1/sqlc/models.go b/payments/db/migration1/sqlc/models.go new file mode 100644 index 000000000..afb448090 --- /dev/null +++ b/payments/db/migration1/sqlc/models.go @@ -0,0 +1,110 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package sqlc + +import ( + "database/sql" + "time" +) + +type Payment struct { + ID int64 + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 +} + +type PaymentAttemptFirstHopCustomRecord struct { + ID int64 + HtlcAttemptIndex int64 + Key int64 + Value []byte +} + +type PaymentDuplicate struct { + ID int64 + PaymentID int64 + AmountMsat int64 + CreatedAt time.Time + FailReason sql.NullInt32 + SettlePreimage []byte + SettleTime sql.NullTime +} + +type PaymentFirstHopCustomRecord struct { + ID int64 + PaymentID int64 + Key int64 + Value []byte +} + +type PaymentHopCustomRecord struct { + ID int64 + HopID int64 + Key int64 + Value []byte +} + +type PaymentHtlcAttempt struct { + ID int64 + AttemptIndex int64 + PaymentID int64 + SessionKey []byte + AttemptTime time.Time + PaymentHash []byte + FirstHopAmountMsat int64 + RouteTotalTimeLock int32 + RouteTotalAmount int64 + RouteSourceKey []byte +} + +type PaymentHtlcAttemptResolution struct { + AttemptIndex int64 + ResolutionTime time.Time + ResolutionType int32 + SettlePreimage []byte + FailureSourceIndex sql.NullInt32 + HtlcFailReason sql.NullInt32 + FailureMsg []byte +} + +type PaymentIntent struct { + ID int64 + PaymentID int64 + IntentType int16 + IntentPayload []byte +} + +type PaymentRouteHop struct { + ID int64 + HtlcAttemptIndex int64 + HopIndex int32 + PubKey []byte + Scid string + OutgoingTimeLock int32 + AmtToForward int64 + MetaData []byte +} + +type PaymentRouteHopAmp struct { + HopID int64 + RootShare []byte + SetID []byte + ChildIndex int32 +} + +type PaymentRouteHopBlinded struct { + HopID int64 + EncryptedData []byte + BlindingPoint []byte + BlindedPathTotalAmt sql.NullInt64 +} + +type PaymentRouteHopMpp struct { + HopID int64 + PaymentAddr []byte + TotalMsat int64 +} diff --git a/payments/db/migration1/sqlc/payments.sql.go b/payments/db/migration1/sqlc/payments.sql.go new file mode 100644 index 000000000..e17bb3cfb --- /dev/null +++ b/payments/db/migration1/sqlc/payments.sql.go @@ -0,0 +1,1357 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: payments.sql + +package sqlc + +import ( + "context" + "database/sql" + "strings" + "time" +) + +const countPayments = `-- name: CountPayments :one +SELECT COUNT(*) FROM payments +` + +func (q *Queries) CountPayments(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countPayments) + var count int64 + err := row.Scan(&count) + return count, err +} + +const deleteFailedAttempts = `-- name: DeleteFailedAttempts :exec +DELETE FROM payment_htlc_attempts WHERE payment_id = $1 AND attempt_index IN ( + SELECT attempt_index FROM payment_htlc_attempt_resolutions WHERE resolution_type = 2 +) +` + +// Delete all failed HTLC attempts for the given payment. Resolution type 2 +// indicates a failed attempt. +func (q *Queries) DeleteFailedAttempts(ctx context.Context, paymentID int64) error { + _, err := q.db.ExecContext(ctx, deleteFailedAttempts, paymentID) + return err +} + +const deletePayment = `-- name: DeletePayment :exec +DELETE FROM payments WHERE id = $1 +` + +func (q *Queries) DeletePayment(ctx context.Context, id int64) error { + _, err := q.db.ExecContext(ctx, deletePayment, id) + return err +} + +const failAttempt = `-- name: FailAttempt :exec +INSERT INTO payment_htlc_attempt_resolutions ( + attempt_index, + resolution_time, + resolution_type, + failure_source_index, + htlc_fail_reason, + failure_msg +) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6 +) +` + +type FailAttemptParams struct { + AttemptIndex int64 + ResolutionTime time.Time + ResolutionType int32 + FailureSourceIndex sql.NullInt32 + HtlcFailReason sql.NullInt32 + FailureMsg []byte +} + +func (q *Queries) FailAttempt(ctx context.Context, arg FailAttemptParams) error { + _, err := q.db.ExecContext(ctx, failAttempt, + arg.AttemptIndex, + arg.ResolutionTime, + arg.ResolutionType, + arg.FailureSourceIndex, + arg.HtlcFailReason, + arg.FailureMsg, + ) + return err +} + +const failPayment = `-- name: FailPayment :execresult +UPDATE payments SET fail_reason = $1 WHERE payment_identifier = $2 +` + +type FailPaymentParams struct { + FailReason sql.NullInt32 + PaymentIdentifier []byte +} + +func (q *Queries) FailPayment(ctx context.Context, arg FailPaymentParams) (sql.Result, error) { + return q.db.ExecContext(ctx, failPayment, arg.FailReason, arg.PaymentIdentifier) +} + +const fetchAllInflightAttempts = `-- name: FetchAllInflightAttempts :many +SELECT + ha.id, + ha.attempt_index, + ha.payment_id, + ha.session_key, + ha.attempt_time, + ha.payment_hash, + ha.first_hop_amount_msat, + ha.route_total_time_lock, + ha.route_total_amount, + ha.route_source_key +FROM payment_htlc_attempts ha +WHERE NOT EXISTS ( + SELECT 1 FROM payment_htlc_attempt_resolutions hr + WHERE hr.attempt_index = ha.attempt_index +) +AND ha.attempt_index > $1 +ORDER BY ha.attempt_index ASC +LIMIT $2 +` + +type FetchAllInflightAttemptsParams struct { + AttemptIndex int64 + Limit int32 +} + +// Fetch all inflight attempts with their payment data using pagination. +// Returns attempt data joined with payment and intent data to avoid separate queries. +func (q *Queries) FetchAllInflightAttempts(ctx context.Context, arg FetchAllInflightAttemptsParams) ([]PaymentHtlcAttempt, error) { + rows, err := q.db.QueryContext(ctx, fetchAllInflightAttempts, arg.AttemptIndex, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentHtlcAttempt + for rows.Next() { + var i PaymentHtlcAttempt + if err := rows.Scan( + &i.ID, + &i.AttemptIndex, + &i.PaymentID, + &i.SessionKey, + &i.AttemptTime, + &i.PaymentHash, + &i.FirstHopAmountMsat, + &i.RouteTotalTimeLock, + &i.RouteTotalAmount, + &i.RouteSourceKey, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchHopLevelCustomRecords = `-- name: FetchHopLevelCustomRecords :many +SELECT + l.id, + l.hop_id, + l.key, + l.value +FROM payment_hop_custom_records l +WHERE l.hop_id IN (/*SLICE:hop_ids*/?) +ORDER BY l.hop_id ASC, l.key ASC +` + +func (q *Queries) FetchHopLevelCustomRecords(ctx context.Context, hopIds []int64) ([]PaymentHopCustomRecord, error) { + query := fetchHopLevelCustomRecords + var queryParams []interface{} + if len(hopIds) > 0 { + for _, v := range hopIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:hop_ids*/?", makeQueryParams(len(queryParams), len(hopIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:hop_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentHopCustomRecord + for rows.Next() { + var i PaymentHopCustomRecord + if err := rows.Scan( + &i.ID, + &i.HopID, + &i.Key, + &i.Value, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchHopsForAttempts = `-- name: FetchHopsForAttempts :many +SELECT + h.id, + h.htlc_attempt_index, + h.hop_index, + h.pub_key, + h.scid, + h.outgoing_time_lock, + h.amt_to_forward, + h.meta_data, + m.payment_addr AS mpp_payment_addr, + m.total_msat AS mpp_total_msat, + a.root_share AS amp_root_share, + a.set_id AS amp_set_id, + a.child_index AS amp_child_index, + b.encrypted_data, + b.blinding_point, + b.blinded_path_total_amt +FROM payment_route_hops h +LEFT JOIN payment_route_hop_mpp m ON m.hop_id = h.id +LEFT JOIN payment_route_hop_amp a ON a.hop_id = h.id +LEFT JOIN payment_route_hop_blinded b ON b.hop_id = h.id +WHERE h.htlc_attempt_index IN (/*SLICE:htlc_attempt_indices*/?) +ORDER BY h.htlc_attempt_index ASC, h.hop_index ASC +` + +type FetchHopsForAttemptsRow struct { + ID int64 + HtlcAttemptIndex int64 + HopIndex int32 + PubKey []byte + Scid string + OutgoingTimeLock int32 + AmtToForward int64 + MetaData []byte + MppPaymentAddr []byte + MppTotalMsat sql.NullInt64 + AmpRootShare []byte + AmpSetID []byte + AmpChildIndex sql.NullInt32 + EncryptedData []byte + BlindingPoint []byte + BlindedPathTotalAmt sql.NullInt64 +} + +func (q *Queries) FetchHopsForAttempts(ctx context.Context, htlcAttemptIndices []int64) ([]FetchHopsForAttemptsRow, error) { + query := fetchHopsForAttempts + var queryParams []interface{} + if len(htlcAttemptIndices) > 0 { + for _, v := range htlcAttemptIndices { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:htlc_attempt_indices*/?", makeQueryParams(len(queryParams), len(htlcAttemptIndices)), 1) + } else { + query = strings.Replace(query, "/*SLICE:htlc_attempt_indices*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchHopsForAttemptsRow + for rows.Next() { + var i FetchHopsForAttemptsRow + if err := rows.Scan( + &i.ID, + &i.HtlcAttemptIndex, + &i.HopIndex, + &i.PubKey, + &i.Scid, + &i.OutgoingTimeLock, + &i.AmtToForward, + &i.MetaData, + &i.MppPaymentAddr, + &i.MppTotalMsat, + &i.AmpRootShare, + &i.AmpSetID, + &i.AmpChildIndex, + &i.EncryptedData, + &i.BlindingPoint, + &i.BlindedPathTotalAmt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchHtlcAttemptResolutionsForPayments = `-- name: FetchHtlcAttemptResolutionsForPayments :many +SELECT + ha.payment_id, + hr.resolution_type +FROM payment_htlc_attempts ha +LEFT JOIN payment_htlc_attempt_resolutions hr ON hr.attempt_index = ha.attempt_index +WHERE ha.payment_id IN (/*SLICE:payment_ids*/?) +` + +type FetchHtlcAttemptResolutionsForPaymentsRow struct { + PaymentID int64 + ResolutionType sql.NullInt32 +} + +// Batch query to fetch only HTLC resolution status for multiple payments. +// We don't need to order by payment_id and attempt_time because we will +// group the resolutions by payment_id in the background. +func (q *Queries) FetchHtlcAttemptResolutionsForPayments(ctx context.Context, paymentIds []int64) ([]FetchHtlcAttemptResolutionsForPaymentsRow, error) { + query := fetchHtlcAttemptResolutionsForPayments + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchHtlcAttemptResolutionsForPaymentsRow + for rows.Next() { + var i FetchHtlcAttemptResolutionsForPaymentsRow + if err := rows.Scan(&i.PaymentID, &i.ResolutionType); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchHtlcAttemptsForPayments = `-- name: FetchHtlcAttemptsForPayments :many +SELECT + ha.id, + ha.attempt_index, + ha.payment_id, + ha.session_key, + ha.attempt_time, + ha.payment_hash, + ha.first_hop_amount_msat, + ha.route_total_time_lock, + ha.route_total_amount, + ha.route_source_key, + hr.resolution_type, + hr.resolution_time, + hr.failure_source_index, + hr.htlc_fail_reason, + hr.failure_msg, + hr.settle_preimage +FROM payment_htlc_attempts ha +LEFT JOIN payment_htlc_attempt_resolutions hr ON hr.attempt_index = ha.attempt_index +WHERE ha.payment_id IN (/*SLICE:payment_ids*/?) +ORDER BY ha.payment_id ASC, ha.attempt_time ASC +` + +type FetchHtlcAttemptsForPaymentsRow struct { + ID int64 + AttemptIndex int64 + PaymentID int64 + SessionKey []byte + AttemptTime time.Time + PaymentHash []byte + FirstHopAmountMsat int64 + RouteTotalTimeLock int32 + RouteTotalAmount int64 + RouteSourceKey []byte + ResolutionType sql.NullInt32 + ResolutionTime sql.NullTime + FailureSourceIndex sql.NullInt32 + HtlcFailReason sql.NullInt32 + FailureMsg []byte + SettlePreimage []byte +} + +func (q *Queries) FetchHtlcAttemptsForPayments(ctx context.Context, paymentIds []int64) ([]FetchHtlcAttemptsForPaymentsRow, error) { + query := fetchHtlcAttemptsForPayments + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchHtlcAttemptsForPaymentsRow + for rows.Next() { + var i FetchHtlcAttemptsForPaymentsRow + if err := rows.Scan( + &i.ID, + &i.AttemptIndex, + &i.PaymentID, + &i.SessionKey, + &i.AttemptTime, + &i.PaymentHash, + &i.FirstHopAmountMsat, + &i.RouteTotalTimeLock, + &i.RouteTotalAmount, + &i.RouteSourceKey, + &i.ResolutionType, + &i.ResolutionTime, + &i.FailureSourceIndex, + &i.HtlcFailReason, + &i.FailureMsg, + &i.SettlePreimage, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchPayment = `-- name: FetchPayment :one +SELECT + p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason, + i.intent_type AS "intent_type", + i.intent_payload AS "intent_payload" +FROM payments p +LEFT JOIN payment_intents i ON i.payment_id = p.id +WHERE p.payment_identifier = $1 +` + +type FetchPaymentRow struct { + Payment Payment + IntentType sql.NullInt16 + IntentPayload []byte +} + +func (q *Queries) FetchPayment(ctx context.Context, paymentIdentifier []byte) (FetchPaymentRow, error) { + row := q.db.QueryRowContext(ctx, fetchPayment, paymentIdentifier) + var i FetchPaymentRow + err := row.Scan( + &i.Payment.ID, + &i.Payment.AmountMsat, + &i.Payment.CreatedAt, + &i.Payment.PaymentIdentifier, + &i.Payment.FailReason, + &i.IntentType, + &i.IntentPayload, + ) + return i, err +} + +const fetchPaymentDuplicates = `-- name: FetchPaymentDuplicates :many +SELECT + id, + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +FROM payment_duplicates +WHERE payment_id = $1 +ORDER BY id ASC +` + +// Fetch all duplicate payment records from the payment_duplicates table for +// a given payment ID. +func (q *Queries) FetchPaymentDuplicates(ctx context.Context, paymentID int64) ([]PaymentDuplicate, error) { + rows, err := q.db.QueryContext(ctx, fetchPaymentDuplicates, paymentID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentDuplicate + for rows.Next() { + var i PaymentDuplicate + if err := rows.Scan( + &i.ID, + &i.PaymentID, + &i.AmountMsat, + &i.CreatedAt, + &i.FailReason, + &i.SettlePreimage, + &i.SettleTime, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchPaymentLevelFirstHopCustomRecords = `-- name: FetchPaymentLevelFirstHopCustomRecords :many +SELECT + l.id, + l.payment_id, + l.key, + l.value +FROM payment_first_hop_custom_records l +WHERE l.payment_id IN (/*SLICE:payment_ids*/?) +ORDER BY l.payment_id ASC, l.key ASC +` + +func (q *Queries) FetchPaymentLevelFirstHopCustomRecords(ctx context.Context, paymentIds []int64) ([]PaymentFirstHopCustomRecord, error) { + query := fetchPaymentLevelFirstHopCustomRecords + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentFirstHopCustomRecord + for rows.Next() { + var i PaymentFirstHopCustomRecord + if err := rows.Scan( + &i.ID, + &i.PaymentID, + &i.Key, + &i.Value, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchPaymentsByIDs = `-- name: FetchPaymentsByIDs :many +SELECT + p.id, + p.amount_msat, + p.created_at, + p.payment_identifier, + p.fail_reason, + pi.intent_type, + pi.intent_payload +FROM payments p +LEFT JOIN payment_intents pi ON pi.payment_id = p.id +WHERE p.id IN (/*SLICE:payment_ids*/?) +ORDER BY p.id ASC +` + +type FetchPaymentsByIDsRow struct { + ID int64 + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 + IntentType sql.NullInt16 + IntentPayload []byte +} + +// Batch fetch payment and intent data for a set of payment IDs. +// Used to avoid fetching redundant payment data when processing multiple +// attempts for the same payment. +func (q *Queries) FetchPaymentsByIDs(ctx context.Context, paymentIds []int64) ([]FetchPaymentsByIDsRow, error) { + query := fetchPaymentsByIDs + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchPaymentsByIDsRow + for rows.Next() { + var i FetchPaymentsByIDsRow + if err := rows.Scan( + &i.ID, + &i.AmountMsat, + &i.CreatedAt, + &i.PaymentIdentifier, + &i.FailReason, + &i.IntentType, + &i.IntentPayload, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchPaymentsByIDsMig = `-- name: FetchPaymentsByIDsMig :many +SELECT + p.id, + p.amount_msat, + p.created_at, + p.payment_identifier, + p.fail_reason, + COUNT(ha.id) AS htlc_attempt_count +FROM payments p +LEFT JOIN payment_htlc_attempts ha ON ha.payment_id = p.id +WHERE p.id IN (/*SLICE:payment_ids*/?) +GROUP BY p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason +ORDER BY p.id ASC +` + +type FetchPaymentsByIDsMigRow struct { + ID int64 + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 + HtlcAttemptCount int64 +} + +// Migration-specific batch fetch that returns payment data along with HTLC +// attempt counts for structural validation during KV to SQL migration. +func (q *Queries) FetchPaymentsByIDsMig(ctx context.Context, paymentIds []int64) ([]FetchPaymentsByIDsMigRow, error) { + query := fetchPaymentsByIDsMig + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchPaymentsByIDsMigRow + for rows.Next() { + var i FetchPaymentsByIDsMigRow + if err := rows.Scan( + &i.ID, + &i.AmountMsat, + &i.CreatedAt, + &i.PaymentIdentifier, + &i.FailReason, + &i.HtlcAttemptCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchRouteLevelFirstHopCustomRecords = `-- name: FetchRouteLevelFirstHopCustomRecords :many +SELECT + l.id, + l.htlc_attempt_index, + l.key, + l.value +FROM payment_attempt_first_hop_custom_records l +WHERE l.htlc_attempt_index IN (/*SLICE:htlc_attempt_indices*/?) +ORDER BY l.htlc_attempt_index ASC, l.key ASC +` + +func (q *Queries) FetchRouteLevelFirstHopCustomRecords(ctx context.Context, htlcAttemptIndices []int64) ([]PaymentAttemptFirstHopCustomRecord, error) { + query := fetchRouteLevelFirstHopCustomRecords + var queryParams []interface{} + if len(htlcAttemptIndices) > 0 { + for _, v := range htlcAttemptIndices { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:htlc_attempt_indices*/?", makeQueryParams(len(queryParams), len(htlcAttemptIndices)), 1) + } else { + query = strings.Replace(query, "/*SLICE:htlc_attempt_indices*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentAttemptFirstHopCustomRecord + for rows.Next() { + var i PaymentAttemptFirstHopCustomRecord + if err := rows.Scan( + &i.ID, + &i.HtlcAttemptIndex, + &i.Key, + &i.Value, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const filterPayments = `-- name: FilterPayments :many +/* ───────────────────────────────────────────── + fetch queries + ───────────────────────────────────────────── +*/ + +SELECT + p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason, + i.intent_type AS "intent_type", + i.intent_payload AS "intent_payload" +FROM payments p +LEFT JOIN payment_intents i ON i.payment_id = p.id +WHERE p.id > COALESCE($1, -1) + AND p.id < COALESCE($2, 9223372036854775807) + -- NOTE: We use non-nullable time params with Go-side defaults instead of + -- COALESCE, because COALESCE with text fallback causes type mismatch on + -- Postgres (timestamp vs text), and OR-based optional filters can prevent + -- the planner from using the created_at index. + AND p.created_at >= $3 + AND p.created_at <= $4 + AND ( + i.intent_type = $5 OR + $5 IS NULL OR i.intent_type IS NULL + ) +ORDER BY p.id ASC +LIMIT $6 +` + +type FilterPaymentsParams struct { + IndexOffsetGet sql.NullInt64 + IndexOffsetLet sql.NullInt64 + CreatedAfter time.Time + CreatedBefore time.Time + IntentType sql.NullInt16 + NumLimit int32 +} + +type FilterPaymentsRow struct { + Payment Payment + IntentType sql.NullInt16 + IntentPayload []byte +} + +func (q *Queries) FilterPayments(ctx context.Context, arg FilterPaymentsParams) ([]FilterPaymentsRow, error) { + rows, err := q.db.QueryContext(ctx, filterPayments, + arg.IndexOffsetGet, + arg.IndexOffsetLet, + arg.CreatedAfter, + arg.CreatedBefore, + arg.IntentType, + arg.NumLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FilterPaymentsRow + for rows.Next() { + var i FilterPaymentsRow + if err := rows.Scan( + &i.Payment.ID, + &i.Payment.AmountMsat, + &i.Payment.CreatedAt, + &i.Payment.PaymentIdentifier, + &i.Payment.FailReason, + &i.IntentType, + &i.IntentPayload, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const filterPaymentsDesc = `-- name: FilterPaymentsDesc :many +SELECT + p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason, + i.intent_type AS "intent_type", + i.intent_payload AS "intent_payload" +FROM payments p +LEFT JOIN payment_intents i ON i.payment_id = p.id +WHERE p.id > COALESCE($1, -1) + AND p.id < COALESCE($2, 9223372036854775807) + -- NOTE: We use non-nullable time params with Go-side defaults instead of + -- COALESCE, because COALESCE with text fallback causes type mismatch on + -- Postgres (timestamp vs text), and OR-based optional filters can prevent + -- the planner from using the created_at index. + AND p.created_at >= $3 + AND p.created_at <= $4 + AND ( + i.intent_type = $5 OR + $5 IS NULL OR i.intent_type IS NULL + ) +ORDER BY p.id DESC +LIMIT $6 +` + +type FilterPaymentsDescParams struct { + IndexOffsetGet sql.NullInt64 + IndexOffsetLet sql.NullInt64 + CreatedAfter time.Time + CreatedBefore time.Time + IntentType sql.NullInt16 + NumLimit int32 +} + +type FilterPaymentsDescRow struct { + Payment Payment + IntentType sql.NullInt16 + IntentPayload []byte +} + +func (q *Queries) FilterPaymentsDesc(ctx context.Context, arg FilterPaymentsDescParams) ([]FilterPaymentsDescRow, error) { + rows, err := q.db.QueryContext(ctx, filterPaymentsDesc, + arg.IndexOffsetGet, + arg.IndexOffsetLet, + arg.CreatedAfter, + arg.CreatedBefore, + arg.IntentType, + arg.NumLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FilterPaymentsDescRow + for rows.Next() { + var i FilterPaymentsDescRow + if err := rows.Scan( + &i.Payment.ID, + &i.Payment.AmountMsat, + &i.Payment.CreatedAt, + &i.Payment.PaymentIdentifier, + &i.Payment.FailReason, + &i.IntentType, + &i.IntentPayload, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const insertHtlcAttempt = `-- name: InsertHtlcAttempt :one +INSERT INTO payment_htlc_attempts ( + payment_id, + attempt_index, + session_key, + attempt_time, + payment_hash, + first_hop_amount_msat, + route_total_time_lock, + route_total_amount, + route_source_key) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9) +RETURNING id +` + +type InsertHtlcAttemptParams struct { + PaymentID int64 + AttemptIndex int64 + SessionKey []byte + AttemptTime time.Time + PaymentHash []byte + FirstHopAmountMsat int64 + RouteTotalTimeLock int32 + RouteTotalAmount int64 + RouteSourceKey []byte +} + +func (q *Queries) InsertHtlcAttempt(ctx context.Context, arg InsertHtlcAttemptParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertHtlcAttempt, + arg.PaymentID, + arg.AttemptIndex, + arg.SessionKey, + arg.AttemptTime, + arg.PaymentHash, + arg.FirstHopAmountMsat, + arg.RouteTotalTimeLock, + arg.RouteTotalAmount, + arg.RouteSourceKey, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertPayment = `-- name: InsertPayment :one +INSERT INTO payments ( + amount_msat, + created_at, + payment_identifier, + fail_reason) +VALUES ( + $1, + $2, + $3, + NULL +) +RETURNING id +` + +type InsertPaymentParams struct { + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte +} + +// Insert a new payment and return its ID. +// When creating a payment we don't have a fail reason because we start the +// payment process. +func (q *Queries) InsertPayment(ctx context.Context, arg InsertPaymentParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPayment, arg.AmountMsat, arg.CreatedAt, arg.PaymentIdentifier) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertPaymentAttemptFirstHopCustomRecord = `-- name: InsertPaymentAttemptFirstHopCustomRecord :exec +INSERT INTO payment_attempt_first_hop_custom_records ( + htlc_attempt_index, + key, + value +) +VALUES ( + $1, + $2, + $3 +) +` + +type InsertPaymentAttemptFirstHopCustomRecordParams struct { + HtlcAttemptIndex int64 + Key int64 + Value []byte +} + +func (q *Queries) InsertPaymentAttemptFirstHopCustomRecord(ctx context.Context, arg InsertPaymentAttemptFirstHopCustomRecordParams) error { + _, err := q.db.ExecContext(ctx, insertPaymentAttemptFirstHopCustomRecord, arg.HtlcAttemptIndex, arg.Key, arg.Value) + return err +} + +const insertPaymentDuplicateMig = `-- name: InsertPaymentDuplicateMig :one +INSERT INTO payment_duplicates ( + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6 +) +RETURNING id +` + +type InsertPaymentDuplicateMigParams struct { + PaymentID int64 + AmountMsat int64 + CreatedAt time.Time + FailReason sql.NullInt32 + SettlePreimage []byte + SettleTime sql.NullTime +} + +// Insert a duplicate payment record into the payment_duplicates table and +// return its ID. +func (q *Queries) InsertPaymentDuplicateMig(ctx context.Context, arg InsertPaymentDuplicateMigParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPaymentDuplicateMig, + arg.PaymentID, + arg.AmountMsat, + arg.CreatedAt, + arg.FailReason, + arg.SettlePreimage, + arg.SettleTime, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertPaymentFirstHopCustomRecord = `-- name: InsertPaymentFirstHopCustomRecord :exec +INSERT INTO payment_first_hop_custom_records ( + payment_id, + key, + value +) +VALUES ( + $1, + $2, + $3 +) +` + +type InsertPaymentFirstHopCustomRecordParams struct { + PaymentID int64 + Key int64 + Value []byte +} + +func (q *Queries) InsertPaymentFirstHopCustomRecord(ctx context.Context, arg InsertPaymentFirstHopCustomRecordParams) error { + _, err := q.db.ExecContext(ctx, insertPaymentFirstHopCustomRecord, arg.PaymentID, arg.Key, arg.Value) + return err +} + +const insertPaymentHopCustomRecord = `-- name: InsertPaymentHopCustomRecord :exec +INSERT INTO payment_hop_custom_records ( + hop_id, + key, + value +) +VALUES ( + $1, + $2, + $3 +) +` + +type InsertPaymentHopCustomRecordParams struct { + HopID int64 + Key int64 + Value []byte +} + +func (q *Queries) InsertPaymentHopCustomRecord(ctx context.Context, arg InsertPaymentHopCustomRecordParams) error { + _, err := q.db.ExecContext(ctx, insertPaymentHopCustomRecord, arg.HopID, arg.Key, arg.Value) + return err +} + +const insertPaymentIntent = `-- name: InsertPaymentIntent :one +INSERT INTO payment_intents ( + payment_id, + intent_type, + intent_payload) +VALUES ( + $1, + $2, + $3 +) +RETURNING id +` + +type InsertPaymentIntentParams struct { + PaymentID int64 + IntentType int16 + IntentPayload []byte +} + +// Insert a payment intent for a given payment and return its ID. +func (q *Queries) InsertPaymentIntent(ctx context.Context, arg InsertPaymentIntentParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPaymentIntent, arg.PaymentID, arg.IntentType, arg.IntentPayload) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertPaymentMig = `-- name: InsertPaymentMig :one +/* ───────────────────────────────────────────── + Migration-specific queries + + These queries are used ONLY for the one-time migration from KV to SQL. + ───────────────────────────────────────────── +*/ + +INSERT INTO payments ( + amount_msat, + created_at, + payment_identifier, + fail_reason) +VALUES ( + $1, + $2, + $3, + $4 +) +RETURNING id +` + +type InsertPaymentMigParams struct { + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 +} + +// Migration-specific payment insert that allows setting fail_reason. +// Normal InsertPayment forces fail_reason to NULL since new payments +// aren't failed yet. During migration, we're inserting historical data +// that may already be failed. +func (q *Queries) InsertPaymentMig(ctx context.Context, arg InsertPaymentMigParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPaymentMig, + arg.AmountMsat, + arg.CreatedAt, + arg.PaymentIdentifier, + arg.FailReason, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertRouteHop = `-- name: InsertRouteHop :one +INSERT INTO payment_route_hops ( + htlc_attempt_index, + hop_index, + pub_key, + scid, + outgoing_time_lock, + amt_to_forward, + meta_data +) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7 +) +RETURNING id +` + +type InsertRouteHopParams struct { + HtlcAttemptIndex int64 + HopIndex int32 + PubKey []byte + Scid string + OutgoingTimeLock int32 + AmtToForward int64 + MetaData []byte +} + +func (q *Queries) InsertRouteHop(ctx context.Context, arg InsertRouteHopParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertRouteHop, + arg.HtlcAttemptIndex, + arg.HopIndex, + arg.PubKey, + arg.Scid, + arg.OutgoingTimeLock, + arg.AmtToForward, + arg.MetaData, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertRouteHopAmp = `-- name: InsertRouteHopAmp :exec +INSERT INTO payment_route_hop_amp ( + hop_id, + root_share, + set_id, + child_index +) +VALUES ( + $1, + $2, + $3, + $4 +) +` + +type InsertRouteHopAmpParams struct { + HopID int64 + RootShare []byte + SetID []byte + ChildIndex int32 +} + +func (q *Queries) InsertRouteHopAmp(ctx context.Context, arg InsertRouteHopAmpParams) error { + _, err := q.db.ExecContext(ctx, insertRouteHopAmp, + arg.HopID, + arg.RootShare, + arg.SetID, + arg.ChildIndex, + ) + return err +} + +const insertRouteHopBlinded = `-- name: InsertRouteHopBlinded :exec +INSERT INTO payment_route_hop_blinded ( + hop_id, + encrypted_data, + blinding_point, + blinded_path_total_amt +) +VALUES ( + $1, + $2, + $3, + $4 +) +` + +type InsertRouteHopBlindedParams struct { + HopID int64 + EncryptedData []byte + BlindingPoint []byte + BlindedPathTotalAmt sql.NullInt64 +} + +func (q *Queries) InsertRouteHopBlinded(ctx context.Context, arg InsertRouteHopBlindedParams) error { + _, err := q.db.ExecContext(ctx, insertRouteHopBlinded, + arg.HopID, + arg.EncryptedData, + arg.BlindingPoint, + arg.BlindedPathTotalAmt, + ) + return err +} + +const insertRouteHopMpp = `-- name: InsertRouteHopMpp :exec +INSERT INTO payment_route_hop_mpp ( + hop_id, + payment_addr, + total_msat +) +VALUES ( + $1, + $2, + $3 +) +` + +type InsertRouteHopMppParams struct { + HopID int64 + PaymentAddr []byte + TotalMsat int64 +} + +func (q *Queries) InsertRouteHopMpp(ctx context.Context, arg InsertRouteHopMppParams) error { + _, err := q.db.ExecContext(ctx, insertRouteHopMpp, arg.HopID, arg.PaymentAddr, arg.TotalMsat) + return err +} + +const settleAttempt = `-- name: SettleAttempt :exec +INSERT INTO payment_htlc_attempt_resolutions ( + attempt_index, + resolution_time, + resolution_type, + settle_preimage +) +VALUES ( + $1, + $2, + $3, + $4 +) +` + +type SettleAttemptParams struct { + AttemptIndex int64 + ResolutionTime time.Time + ResolutionType int32 + SettlePreimage []byte +} + +func (q *Queries) SettleAttempt(ctx context.Context, arg SettleAttemptParams) error { + _, err := q.db.ExecContext(ctx, settleAttempt, + arg.AttemptIndex, + arg.ResolutionTime, + arg.ResolutionType, + arg.SettlePreimage, + ) + return err +} From a9747207f6a5aab2a3723027b16f41e6898d3a1b Mon Sep 17 00:00:00 2001 From: ziggie Date: Sun, 11 Jan 2026 00:02:52 +0100 Subject: [PATCH 04/16] payments/migration1: add the payments mig code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the KV→SQL payment migration and add an in-migration validation pass that deep-compares KV and SQL payment data in batches. Duplicate payments are migrated into the payment_duplicates table, and duplicates without attempt info or explicit resolution are marked failed to ensure terminal state. Validation checks those rows as well. --- .../db/migration1/migration_validation.go | 555 ++++++++++++ payments/db/migration1/sql_migration.go | 817 ++++++++++++++++++ 2 files changed, 1372 insertions(+) create mode 100644 payments/db/migration1/migration_validation.go create mode 100644 payments/db/migration1/sql_migration.go diff --git a/payments/db/migration1/migration_validation.go b/payments/db/migration1/migration_validation.go new file mode 100644 index 000000000..e97c3db40 --- /dev/null +++ b/payments/db/migration1/migration_validation.go @@ -0,0 +1,555 @@ +package migration1 + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "reflect" + "sort" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/record" + "github.com/pmezard/go-difflib/difflib" +) + +// migratedPaymentRef is a reference to a migrated payment. +type migratedPaymentRef struct { + Hash lntypes.Hash + PaymentID int64 +} + +// validateMigratedPaymentBatch performs a structural validation pass by +// comparing key fields (hash, amount, fail reason, HTLC count) of KV payments +// with their SQL counterparts. If a structural mismatch is detected, a full +// deep comparison is performed to produce a detailed diff for debugging. +func validateMigratedPaymentBatch(ctx context.Context, + kvBackend kvdb.Backend, sqlDB SQLQueries, + cfg *SQLStoreConfig, batch []migratedPaymentRef) error { + + if len(batch) == 0 { + return nil + } + + if cfg == nil || cfg.QueryCfg == nil { + return fmt.Errorf("missing SQL store config for validation") + } + + paymentIDs := make([]int64, 0, len(batch)) + for _, item := range batch { + paymentIDs = append(paymentIDs, item.PaymentID) + } + + rows, err := sqlDB.FetchPaymentsByIDsMig(ctx, paymentIDs) + if err != nil { + return fmt.Errorf("fetch SQL payments: %w", err) + } + if len(rows) != len(paymentIDs) { + return fmt.Errorf("SQL payment batch mismatch: got=%d want=%d", + len(rows), len(paymentIDs)) + } + + // Perform the structural check by comparing key fields from the KV + // store with the SQL store. + err = kvBackend.View(func(kvTx kvdb.RTx) error { + paymentsBucket := kvTx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + return fmt.Errorf("no payments bucket") + } + + for _, row := range rows { + hash := row.PaymentIdentifier + var paymentHash lntypes.Hash + copy(paymentHash[:], hash) + + paymentBucket := paymentsBucket.NestedReadBucket(hash) + if paymentBucket == nil { + return fmt.Errorf("missing payment bucket %x", + hash[:8]) + } + + kvPayment, err := fetchPayment(paymentBucket) + if err != nil { + return fmt.Errorf("fetch KV payment %x: %w", + hash[:8], err) + } + + err = structuralCompare(kvPayment, row) + if err != nil { + // On structural mismatch, perform a deep + // comparison to produce a detailed diff. + deepErr := deepComparePayment( + ctx, cfg, sqlDB, row.ID, + paymentHash, kvPayment, + ) + if deepErr != nil { + return deepErr + } + + // If deep comparison passes but structural + // failed, report the structural error as it + // indicates an unexpected inconsistency. + return err + } + + err = compareDuplicatePayments( + ctx, paymentBucket, sqlDB, row.ID, + paymentHash, + ) + if err != nil { + return err + } + } + + return nil + }, func() {}) + if err != nil { + return err + } + + return nil +} + +// structuralCompare performs a fast structural comparison between a KV payment +// and a SQL payment row, checking key fields: payment identifier, amount, +// failure reason, and HTLC attempt count. +func structuralCompare(kvPayment *MPPayment, + sqlRow sqlc.FetchPaymentsByIDsMigRow) error { + + // Compare payment identifier. + kvHash := kvPayment.Info.PaymentIdentifier[:] + if !bytes.Equal(kvHash, sqlRow.PaymentIdentifier) { + return fmt.Errorf("payment identifier mismatch: kv=%x sql=%x", + kvHash[:8], sqlRow.PaymentIdentifier[:8]) + } + + // Compare amount. + kvAmount := int64(kvPayment.Info.Value) + if kvAmount != sqlRow.AmountMsat { + return fmt.Errorf("amount mismatch for %x: kv=%d sql=%d", + sqlRow.PaymentIdentifier[:8], kvAmount, + sqlRow.AmountMsat) + } + + // Compare failure reason. + var kvFailReason sql.NullInt32 + if kvPayment.FailureReason != nil { + kvFailReason = sql.NullInt32{ + Int32: int32(*kvPayment.FailureReason), + Valid: true, + } + } + if kvFailReason != sqlRow.FailReason { + return fmt.Errorf("fail reason mismatch for %x: kv=%v sql=%v", + sqlRow.PaymentIdentifier[:8], kvFailReason, + sqlRow.FailReason) + } + + // Compare HTLC attempt count. + kvHTLCCount := int64(len(kvPayment.HTLCs)) + if kvHTLCCount != sqlRow.HtlcAttemptCount { + return fmt.Errorf("HTLC count mismatch for %x: kv=%d sql=%d", + sqlRow.PaymentIdentifier[:8], kvHTLCCount, + sqlRow.HtlcAttemptCount) + } + + return nil +} + +// deepComparePayment performs a full deep comparison between a KV payment and +// its SQL counterpart, producing a detailed diff on mismatch. +func deepComparePayment(ctx context.Context, cfg *SQLStoreConfig, + sqlDB SQLQueries, paymentID int64, paymentHash lntypes.Hash, + kvPayment *MPPayment) error { + + batchData, err := batchLoadPaymentDetailsData( + ctx, cfg.QueryCfg, sqlDB, []int64{paymentID}, + ) + if err != nil { + return fmt.Errorf("load payment data for deep compare %x: %w", + paymentHash[:8], err) + } + + byIDRows, err := sqlDB.FetchPaymentsByIDs(ctx, []int64{paymentID}) + if err != nil { + return fmt.Errorf("fetch payment by ID for deep compare "+ + "%x: %w", paymentHash[:8], err) + } + if len(byIDRows) != 1 { + return fmt.Errorf("expected 1 payment for deep compare, "+ + "got %d", len(byIDRows)) + } + + sqlPayment, err := buildPaymentFromBatchData(byIDRows[0], batchData) + if err != nil { + return fmt.Errorf("build SQL payment %x: %w", + paymentHash[:8], err) + } + + normalizePaymentForCompare(kvPayment) + normalizePaymentForCompare(sqlPayment) + + if !reflect.DeepEqual(kvPayment, sqlPayment) { + dumpCfg := spew.ConfigState{ + DisablePointerAddresses: true, + DisableCapacities: true, + DisableMethods: true, + SortKeys: true, + } + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines( + dumpCfg.Sdump(kvPayment), + ), + B: difflib.SplitLines( + dumpCfg.Sdump(sqlPayment), + ), + FromFile: "kv", + ToFile: "sql", + Context: 3, + } + diffText, _ := difflib.GetUnifiedDiffString(diff) + + return fmt.Errorf("payment mismatch %x\n%s", + paymentHash[:8], diffText) + } + + return nil +} + +// normalizePaymentForCompare normalizes fields that are expected to differ +// between KV and SQL representations before deep comparison. +func normalizePaymentForCompare(payment *MPPayment) { + if payment == nil { + return + } + + // SequenceNum will not be equal because the kv db can have already + // payments deleted during its lifetime. + payment.SequenceNum = 0 + + // We normalize timestamps before deep-comparing KV vs SQL objects. + // + // - **Microseconds**: SQL backends typically persist timestamps at + // microsecond precision (e.g. Postgres), while KV (Go `time.Time`) + // can contain nanoseconds. Truncating avoids false mismatches caused + // solely by differing storage precision. + // + // - **Local timezone**: when reading from SQL, timestamps are typically + // materialized in the local timezone by the SQL layer (and/or + // converters). Converting both sides to `time.Local` ensures the + // comparison is consistent across KV and SQL representations. + trunc := func(t time.Time) time.Time { + if t.IsZero() { + return t + } + + return time.Unix(0, t.UnixNano()). + In(time.Local). + Truncate(time.Microsecond) + } + + // Normalize PaymentCreationInfo fields. + if payment.Info != nil { + payment.Info.CreationTime = trunc( + payment.Info.CreationTime, + ) + if len(payment.Info.PaymentRequest) == 0 { + payment.Info.PaymentRequest = []byte{} + } + if len(payment.Info.FirstHopCustomRecords) == 0 { + payment.Info.FirstHopCustomRecords = lnwire. + CustomRecords{} + } + } + + // Normalize HTLCAttemptInfo so nil is converted to an empty slice. + if len(payment.HTLCs) == 0 { + payment.HTLCs = []HTLCAttempt{} + } + + // Normalize HTLC attempt ordering; SQL/KV may return attempts + // in different orders. + sort.SliceStable(payment.HTLCs, func(i, j int) bool { + return payment.HTLCs[i].AttemptID < payment.HTLCs[j].AttemptID + }) + + // Normalize HTLCAttemptInfo fields. + for i := range payment.HTLCs { + htlc := &payment.HTLCs[i] + + htlc.AttemptTime = trunc(htlc.AttemptTime) + if htlc.Settle != nil { + htlc.Settle.SettleTime = trunc( + htlc.Settle.SettleTime, + ) + } + if htlc.Failure != nil { + htlc.Failure.FailTime = trunc( + htlc.Failure.FailTime, + ) + } + + // Clear cached fields not persisted in storage. + htlc.onionBlob = [1366]byte{} + htlc.circuit = nil + htlc.cachedSessionKey = nil + + if len(htlc.Route.FirstHopWireCustomRecords) == 0 { + htlc.Route.FirstHopWireCustomRecords = + lnwire.CustomRecords{} + } + + for j := range htlc.Route.Hops { + if len(htlc.Route.Hops[j].CustomRecords) == 0 { + htlc.Route.Hops[j].CustomRecords = + record.CustomSet{} + } + } + } +} + +// duplicateRecord is a record that represents a duplicate payment. +type duplicateRecord struct { + AmountMsat int64 + CreatedAt time.Time + FailReason sql.NullInt32 + SettlePreimage []byte + SettleTime sql.NullTime +} + +// compareDuplicatePayments validates migrated duplicate rows against KV data. +func compareDuplicatePayments(ctx context.Context, paymentBucket kvdb.RBucket, + sqlDB SQLQueries, paymentID int64, hash lntypes.Hash) error { + + // Fetch the duplicate payments from the KV store. + kvDuplicates, err := fetchDuplicateRecords(paymentBucket) + if err != nil { + return fmt.Errorf("fetch KV duplicates %x: %w", + hash[:8], err) + } + + // Fetch the duplicate payments from the SQL store. + sqlDuplicates, err := sqlDB.FetchPaymentDuplicates(ctx, paymentID) + if err != nil { + return fmt.Errorf("fetch SQL duplicates %x: %w", + hash[:8], err) + } + + if len(kvDuplicates) != len(sqlDuplicates) { + return fmt.Errorf("duplicate count mismatch %x: kv=%d "+ + "sql=%d", hash[:8], len(kvDuplicates), + len(sqlDuplicates)) + } + + kvNormalized := normalizeDuplicateRecords(kvDuplicates) + sqlNormalized := normalizeDuplicateRecords( + dbDuplicatesToDuplicateRecords(sqlDuplicates), + ) + + sortDuplicates(kvNormalized) + sortDuplicates(sqlNormalized) + + if !reflect.DeepEqual(kvNormalized, sqlNormalized) { + dumpCfg := spew.ConfigState{ + DisablePointerAddresses: true, + DisableCapacities: true, + DisableMethods: true, + SortKeys: true, + } + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines( + dumpCfg.Sdump(kvNormalized), + ), + B: difflib.SplitLines( + dumpCfg.Sdump(sqlNormalized), + ), + FromFile: "kv", + ToFile: "sql", + Context: 3, + } + diffText, _ := difflib.GetUnifiedDiffString(diff) + + return fmt.Errorf("duplicate mismatch %x\n%s", + hash[:8], diffText) + } + + return nil +} + +// fetchDuplicateRecords reads duplicate payment records from the KV bucket. +func fetchDuplicateRecords(paymentBucket kvdb.RBucket) ([]duplicateRecord, + error) { + + dupBucket := paymentBucket.NestedReadBucket(duplicatePaymentsBucket) + if dupBucket == nil { + return nil, nil + } + + var duplicates []duplicateRecord + err := dupBucket.ForEach(func(seqBytes, _ []byte) error { + if len(seqBytes) != 8 { + return nil + } + + subBucket := dupBucket.NestedReadBucket(seqBytes) + if subBucket == nil { + return nil + } + + creationData := subBucket.Get(duplicatePaymentCreationInfoKey) + if creationData == nil { + return fmt.Errorf("missing duplicate creation info") + } + + creationInfo, err := deserializeDuplicatePaymentCreationInfo( + bytes.NewReader(creationData), + ) + if err != nil { + return fmt.Errorf("deserialize duplicate creation "+ + "info: %w", err) + } + + settleData := subBucket.Get(duplicatePaymentSettleInfoKey) + failReasonData := subBucket.Get(duplicatePaymentFailInfoKey) + + if settleData != nil && len(failReasonData) > 0 { + return fmt.Errorf("duplicate has both settle and " + + "fail info") + } + + var ( + failReason sql.NullInt32 + settlePreimage []byte + settleTime sql.NullTime + ) + + switch { + case settleData != nil: + settlePreimage, settleTime, err = + parseDuplicateSettleData(settleData) + if err != nil { + return err + } + case len(failReasonData) > 0: + failReason = sql.NullInt32{ + Int32: int32(failReasonData[0]), + Valid: true, + } + default: + // If the duplicate has no settle or fail info, it is + // considered failed. Every duplicate payment must have + // either a settle or fail info in the sql database. So + // we set the fail reason to error to mimic the behavior + // for the kv store. + failReason = sql.NullInt32{ + Int32: int32(FailureReasonError), + Valid: true, + } + } + + duplicates = append(duplicates, duplicateRecord{ + AmountMsat: int64(creationInfo.Value), + CreatedAt: normalizeTimeForSQL( + creationInfo.CreationTime, + ), + FailReason: failReason, + SettlePreimage: settlePreimage, + SettleTime: settleTime, + }) + + return nil + }) + if err != nil { + return nil, err + } + + return duplicates, nil +} + +// dbDuplicatesToDuplicateRecords maps SQL duplicate rows into comparable +// duplicate records. +func dbDuplicatesToDuplicateRecords( + rows []sqlc.PaymentDuplicate) []duplicateRecord { + + duplicates := make([]duplicateRecord, 0, len(rows)) + for _, row := range rows { + duplicates = append(duplicates, duplicateRecord{ + AmountMsat: row.AmountMsat, + CreatedAt: row.CreatedAt, + FailReason: row.FailReason, + SettlePreimage: row.SettlePreimage, + SettleTime: row.SettleTime, + }) + } + + return duplicates +} + +// normalizeDuplicateRecords normalizes time precision and empty fields. +func normalizeDuplicateRecords(records []duplicateRecord) []duplicateRecord { + if len(records) == 0 { + return []duplicateRecord{} + } + + trunc := func(t time.Time) time.Time { + if t.IsZero() { + return t + } + + return t.In(time.Local).Truncate(time.Microsecond) + } + + for i := range records { + records[i].CreatedAt = trunc(records[i].CreatedAt) + if records[i].SettleTime.Valid { + records[i].SettleTime.Time = trunc( + records[i].SettleTime.Time, + ) + } + + if len(records[i].SettlePreimage) == 0 { + records[i].SettlePreimage = []byte{} + } + } + + return records +} + +// sortDuplicates orders records deterministically for deep comparison. +func sortDuplicates(records []duplicateRecord) { + sort.SliceStable(records, func(i, j int) bool { + ai := records[i] + aj := records[j] + + // Duplicates are "duplicates" because they share the same + // payment identifier. So ordering can be stable using + // timestamp + amount. + if !ai.CreatedAt.Equal(aj.CreatedAt) { + return ai.CreatedAt.Before(aj.CreatedAt) + } + + return ai.AmountMsat < aj.AmountMsat + }) +} + +// validatePaymentCounts compares the number of migrated payments with the SQL +// payment count to catch missing rows. +func validatePaymentCounts(ctx context.Context, sqlDB SQLQueries, + expectedCount int64) error { + + sqlCount, err := sqlDB.CountPayments(ctx) + if err != nil { + return fmt.Errorf("count SQL payments: %w", err) + } + if expectedCount != sqlCount { + return fmt.Errorf("payment count mismatch: kv=%d sql=%d", + expectedCount, sqlCount) + } + + return nil +} diff --git a/payments/db/migration1/sql_migration.go b/payments/db/migration1/sql_migration.go new file mode 100644 index 000000000..a7a02af5b --- /dev/null +++ b/payments/db/migration1/sql_migration.go @@ -0,0 +1,817 @@ +package migration1 + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "strconv" + "time" + + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "golang.org/x/time/rate" +) + +// MigrationStats tracks migration progress. +type MigrationStats struct { + TotalPayments int64 + SuccessfulPayments int64 + FailedPayments int64 + InFlightPayments int64 + InitiatedPayments int64 + TotalAttempts int64 + SettledAttempts int64 + FailedAttempts int64 + InFlightAttempts int64 + TotalHops int64 + DuplicatePayments int64 + DuplicateEntries int64 + SkippedPayments int64 + MigrationDuration time.Duration +} + +// MigratePaymentsKVToSQL migrates payments from KV to SQL and validates +// migrated data in batches. Callers are responsible for executing this within +// a single SQL transaction if atomicity is required. +func MigratePaymentsKVToSQL(ctx context.Context, kvBackend kvdb.Backend, + sqlDB SQLQueries, cfg *SQLStoreConfig) error { + + if cfg == nil { + return fmt.Errorf("missing SQL store config for migration") + } + + if cfg.QueryCfg == nil { + return fmt.Errorf("missing SQL store config for " + + "validation") + } + + if cfg.QueryCfg.MaxBatchSize == 0 { + return fmt.Errorf("invalid max batch size for " + + "validation") + } + + stats := &MigrationStats{} + startTime := time.Now() + + log.Infof("Starting payment migration from KV to SQL...") + + var ( + validationBatch []migratedPaymentRef + validatedPayments int64 + + reportInterval = rate.Sometimes{Interval: 5 * time.Second} + validationInterval = rate.Sometimes{Interval: 5 * time.Second} + ) + + // Open the KV backend in read-only mode. + err := kvBackend.View(func(kvTx kvdb.RTx) error { + // In case we start with an empty database, there are no + // payments to migrate. + paymentsBucket := kvTx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + log.Infof("No payments bucket found - database is " + + "empty") + + return nil + } + + // The index bucket maps sequence number -> payment hash. + indexes := kvTx.ReadBucket(paymentsIndexBucket) + if indexes == nil { + return fmt.Errorf("index bucket does not exist") + } + + // We iterate over all sequence numbers in the index bucket to + // make sure we have the correct order of payments. Otherwise, + // if we just loop over the payments bucket, we might get the + // payments not in the chronological order but rather the + // lexicographical order of the payment hashes. + return indexes.ForEach(func(seqKey, indexVal []byte) error { + // Progress reporting based on time + actual work done. + reportProgress := func() { + elapsed := time.Since(startTime) + if elapsed <= 0 { + return + } + + if stats.TotalPayments == 0 { + return + } + + paymentRate := float64(stats.TotalPayments) / + elapsed.Seconds() + attemptRate := float64(stats.TotalAttempts) / + elapsed.Seconds() + + log.Infof("Progress: %d payments, %d "+ + "attempts, %d hops | Rate: %.1f "+ + "pmt/s, %.1f att/s | Elapsed: %v", + stats.TotalPayments, + stats.TotalAttempts, stats.TotalHops, + paymentRate, attemptRate, + elapsed.Round(time.Second), + ) + } + + reportInterval.Do(reportProgress) + + r := bytes.NewReader(indexVal) + paymentHash, err := deserializePaymentIndex(r) + if err != nil { + return err + } + + paymentBucket := paymentsBucket.NestedReadBucket( + paymentHash[:], + ) + if paymentBucket == nil { + // We skip the entry in case this sequence + // number does not have a corresponding + // payment bucket. But aborting would not help + // either because it is just a db inconsistency. + log.Warnf("Missing bucket for payment %x", + paymentHash[:8]) + + stats.SkippedPayments++ + + return nil + } + + // Every payment bucket should have a sequence number + // which is also important to check for duplicates. + seqBytes := paymentBucket.Get(paymentSequenceKey) + if seqBytes == nil { + return ErrNoSequenceNumber + } + + // Skip duplicates. They are migrated into the + // payment_duplicates table when the primary payment is + // processed. + if !bytes.Equal(seqBytes, seqKey) { + return nil + } + + // Fetch the payment from the kv store. + payment, err := fetchPayment(paymentBucket) + if err != nil { + return fmt.Errorf("fetch payment %x: %w", + paymentHash[:8], err) + } + + // Migrate the payment to the SQL database. + paymentID, err := migratePayment( + ctx, payment, paymentHash, sqlDB, stats, + ) + if err != nil { + return fmt.Errorf("migrate payment %x: %w", + paymentHash[:8], err) + } + + // Check for duplicates. + dupBucket := paymentBucket.NestedReadBucket( + duplicatePaymentsBucket, + ) + if dupBucket != nil { + err = migrateDuplicatePayments( + ctx, dupBucket, paymentHash, + paymentID, sqlDB, stats, + ) + if err != nil { + return fmt.Errorf("migrate duplicates "+ + "%x: %w", paymentHash[:8], + err) + } + } + + // Add the payment to the validation batch. + validationBatch = append( + validationBatch, migratedPaymentRef{ + Hash: paymentHash, + PaymentID: paymentID, + }, + ) + if uint32(len(validationBatch)) >= + cfg.QueryCfg.MaxBatchSize { + + err := validateMigratedPaymentBatch( + ctx, kvBackend, sqlDB, + cfg, + validationBatch, + ) + if err != nil { + return err + } + + validatedPayments += int64( + len(validationBatch), + ) + + // Log validation progress periodically. + validationInterval.Do(func() { + log.Infof("Validated %d/%d "+ + "payments", + validatedPayments, + stats.TotalPayments, + ) + }) + + validationBatch = validationBatch[:0] + } + + return nil + }) + }, func() {}) + + if err != nil { + return fmt.Errorf("migrate payments: %w", err) + } + + // Validate any remaining payments in the batch. + if len(validationBatch) > 0 { + if err := validateMigratedPaymentBatch( + ctx, kvBackend, sqlDB, cfg, validationBatch, + ); err != nil { + return err + } + + validatedPayments += int64(len(validationBatch)) + log.Infof("Validated %d/%d payments", validatedPayments, + stats.TotalPayments) + } + + // Validate the total number of payments as an additional sanity check. + if err := validatePaymentCounts( + ctx, sqlDB, stats.TotalPayments, + ); err != nil { + return err + } + + stats.MigrationDuration = time.Since(startTime) + + printMigrationSummary(stats) + + return nil +} + +// normalizeTimeForSQL converts a timestamp into the representation we persist +// and compare against in SQL: +// - drops any monotonic clock reading (SQL can't store it), +// - forces UTC for deterministic comparisons across environments. +// +// A zero time is returned unchanged. +func normalizeTimeForSQL(t time.Time) time.Time { + if t.IsZero() { + return t + } + + return time.Unix(0, t.UnixNano()).UTC() +} + +// migratePayment migrates a single payment from KV to SQL. +func migratePayment(ctx context.Context, payment *MPPayment, hash lntypes.Hash, + sqlDB SQLQueries, stats *MigrationStats) (int64, error) { + + // Update migration stats based on payment status. + switch payment.Status { + case StatusSucceeded: + stats.SuccessfulPayments++ + + case StatusFailed: + stats.FailedPayments++ + + case StatusInFlight: + stats.InFlightPayments++ + + case StatusInitiated: + stats.InitiatedPayments++ + } + + // Prepare fail reason for SQL insert. + var failReason sql.NullInt32 + if payment.FailureReason != nil { + failReason = sql.NullInt32{ + Int32: int32(*payment.FailureReason), + Valid: true, + } + } + + // Insert payment using migration query. + paymentID, err := sqlDB.InsertPaymentMig( + ctx, sqlc.InsertPaymentMigParams{ + AmountMsat: int64(payment.Info.Value), + CreatedAt: normalizeTimeForSQL( + payment.Info.CreationTime, + ), + PaymentIdentifier: hash[:], + FailReason: failReason, + }) + if err != nil { + return 0, fmt.Errorf("insert payment: %w", err) + } + + // Insert payment intent. + // + // Only insert a row if we have an actual intent payload. For legacy + // hash-only/keysend-style payments, the intent may be absent. + if len(payment.Info.PaymentRequest) > 0 { + _, err = sqlDB.InsertPaymentIntent( + ctx, sqlc.InsertPaymentIntentParams{ + PaymentID: paymentID, + IntentType: int16(PaymentIntentTypeBolt11), + IntentPayload: payment.Info.PaymentRequest, + }, + ) + if err != nil { + return 0, fmt.Errorf("insert intent: %w", err) + } + } + + // Insert first hop custom records (payment level). + for key, value := range payment.Info.FirstHopCustomRecords { + err = sqlDB.InsertPaymentFirstHopCustomRecord(ctx, + sqlc.InsertPaymentFirstHopCustomRecordParams{ + PaymentID: paymentID, + Key: int64(key), + Value: value, + }, + ) + if err != nil { + return 0, fmt.Errorf("insert custom record: %w", err) + } + } + + // Migrate HTLC attempts. + for _, htlc := range payment.HTLCs { + err = migrateHTLCAttempt( + ctx, paymentID, hash, &htlc, sqlDB, stats, + ) + if err != nil { + return 0, fmt.Errorf("migrate attempt %d: %w", + htlc.AttemptID, err) + } + } + + stats.TotalPayments++ + + return paymentID, nil +} + +// migrateHTLCAttempt migrates a single HTLC attempt. +func migrateHTLCAttempt(ctx context.Context, paymentID int64, + parentPaymentHash lntypes.Hash, htlc *HTLCAttempt, sqlDB SQLQueries, + stats *MigrationStats) error { + + // Validate that we have a payment hash for the attempt. + // + // NOTE: We always require an attempt payment hash. A missing hash is an + // unrecoverable inconsistency. All payments should have a payment hash + // (AMP,MPP,Legacy) + var paymentHash []byte + switch { + case htlc.Hash != nil: + paymentHash = (*htlc.Hash)[:] + + default: + return fmt.Errorf("HTLC attempt %d missing payment hash "+ + "(parent payment hash=%x)", htlc.AttemptID, + parentPaymentHash[:]) + } + + firstHopAmountMsat := int64(htlc.Route.FirstHopAmount.Val.Int()) + + // Get the session key bytes. + sessionKeyBytes := htlc.SessionKey().Serialize() + + // Insert HTLC attempt. + _, err := sqlDB.InsertHtlcAttempt(ctx, sqlc.InsertHtlcAttemptParams{ + PaymentID: paymentID, + AttemptIndex: int64(htlc.AttemptID), + SessionKey: sessionKeyBytes, + AttemptTime: normalizeTimeForSQL(htlc.AttemptTime), + PaymentHash: paymentHash, + FirstHopAmountMsat: firstHopAmountMsat, + RouteTotalTimeLock: int32(htlc.Route.TotalTimeLock), + RouteTotalAmount: int64(htlc.Route.TotalAmount), + RouteSourceKey: htlc.Route.SourcePubKey[:], + }) + if err != nil { + return fmt.Errorf("insert HTLC attempt: %w", err) + } + + // Insert the route-level first hop custom records. + for key, value := range htlc.Route.FirstHopWireCustomRecords { + err = sqlDB.InsertPaymentAttemptFirstHopCustomRecord( + ctx, + sqlc.InsertPaymentAttemptFirstHopCustomRecordParams{ + HtlcAttemptIndex: int64(htlc.AttemptID), + Key: int64(key), + Value: value, + }, + ) + if err != nil { + return fmt.Errorf("insert attempt first hop custom "+ + "record: %w", err) + } + } + + // Insert route hops. + for hopIndex := range htlc.Route.Hops { + hop := htlc.Route.Hops[hopIndex] + err = migrateRouteHop( + ctx, int64(htlc.AttemptID), hopIndex, hop, + sqlDB, stats, + ) + if err != nil { + return fmt.Errorf("migrate hop %d: %w", hopIndex, err) + } + } + + // Handle attempt resolution (settle or fail). + switch { + case htlc.Settle != nil: + // Settled + err = sqlDB.SettleAttempt(ctx, sqlc.SettleAttemptParams{ + AttemptIndex: int64(htlc.AttemptID), + ResolutionTime: normalizeTimeForSQL( + htlc.Settle.SettleTime, + ), + ResolutionType: int32(HTLCAttemptResolutionSettled), + SettlePreimage: htlc.Settle.Preimage[:], + }) + if err != nil { + return fmt.Errorf("settle attempt: %w", err) + } + + stats.SettledAttempts++ + + case htlc.Failure != nil: + var failureMsg bytes.Buffer + if htlc.Failure.Message != nil { + err := lnwire.EncodeFailureMessage( + &failureMsg, htlc.Failure.Message, 0, + ) + if err != nil { + return fmt.Errorf("failed to encode "+ + "failure message: %w", err) + } + } + + err = sqlDB.FailAttempt(ctx, sqlc.FailAttemptParams{ + AttemptIndex: int64(htlc.AttemptID), + ResolutionTime: normalizeTimeForSQL( + htlc.Failure.FailTime, + ), + ResolutionType: int32(HTLCAttemptResolutionFailed), + FailureSourceIndex: sql.NullInt32{ + Int32: int32(htlc.Failure.FailureSourceIndex), + Valid: true, + }, + HtlcFailReason: sql.NullInt32{ + Int32: int32(htlc.Failure.Reason), + Valid: true, + }, + FailureMsg: failureMsg.Bytes(), + }) + if err != nil { + return fmt.Errorf("fail attempt: %w", err) + } + + stats.FailedAttempts++ + + default: + // If the attempt is not settled or failed, it is in flight. + stats.InFlightAttempts++ + } + + stats.TotalAttempts++ + + return nil +} + +// migrateRouteHop migrates a single route hop. +func migrateRouteHop(ctx context.Context, + attemptID int64, hopIndex int, hop *Hop, sqlDB SQLQueries, + stats *MigrationStats) error { + + // Convert channel ID to string representation of uint64. + // The SCID is stored as a decimal string to match the converter + // expectations (sql_converters.go:173). + scidStr := strconv.FormatUint(hop.ChannelID, 10) + + // Insert route hop. + hopID, err := sqlDB.InsertRouteHop(ctx, sqlc.InsertRouteHopParams{ + HtlcAttemptIndex: attemptID, + HopIndex: int32(hopIndex), + PubKey: hop.PubKeyBytes[:], + Scid: scidStr, + OutgoingTimeLock: int32(hop.OutgoingTimeLock), + AmtToForward: int64(hop.AmtToForward), + MetaData: hop.Metadata, + }) + if err != nil { + return fmt.Errorf("insert hop: %w", err) + } + + // Check for blinded route data (route blinding). + if len(hop.EncryptedData) > 0 || hop.BlindingPoint != nil || + hop.TotalAmtMsat != 0 { + + var blindingPoint []byte + if hop.BlindingPoint != nil { + blindingPoint = hop.BlindingPoint.SerializeCompressed() + } + + var totalAmt sql.NullInt64 + if hop.TotalAmtMsat != 0 { + totalAmt = sql.NullInt64{ + Int64: int64(hop.TotalAmtMsat), + Valid: true, + } + } + + err := sqlDB.InsertRouteHopBlinded( + ctx, sqlc.InsertRouteHopBlindedParams{ + HopID: hopID, + EncryptedData: hop.EncryptedData, + BlindingPoint: blindingPoint, + BlindedPathTotalAmt: totalAmt, + }, + ) + if err != nil { + return fmt.Errorf("insert blinded hop: %w", err) + } + } + + // Check for MPP record. + if hop.MPP != nil { + paymentAddr := hop.MPP.PaymentAddr() + err = sqlDB.InsertRouteHopMpp(ctx, sqlc.InsertRouteHopMppParams{ + HopID: hopID, + PaymentAddr: paymentAddr[:], + TotalMsat: int64(hop.MPP.TotalMsat()), + }) + if err != nil { + return fmt.Errorf("insert MPP: %w", err) + } + } + + // Check for AMP record. + if hop.AMP != nil { + rootShare := hop.AMP.RootShare() + setID := hop.AMP.SetID() + err = sqlDB.InsertRouteHopAmp(ctx, sqlc.InsertRouteHopAmpParams{ + HopID: hopID, + RootShare: rootShare[:], + SetID: setID[:], + ChildIndex: int32(hop.AMP.ChildIndex()), + }) + if err != nil { + return fmt.Errorf("insert AMP: %w", err) + } + } + + // Check for custom records. + if hop.CustomRecords != nil { + for tlvType, value := range hop.CustomRecords { + err = sqlDB.InsertPaymentHopCustomRecord( + ctx, + sqlc.InsertPaymentHopCustomRecordParams{ + HopID: hopID, + Key: int64(tlvType), + Value: value, + }, + ) + if err != nil { + return fmt.Errorf("insert hop custom "+ + "record: %w", err) + } + } + } + + stats.TotalHops++ + + return nil +} + +// migrateDuplicatePayments migrates duplicate payments into the dedicated +// payment_duplicates table. +func migrateDuplicatePayments(ctx context.Context, dupBucket kvdb.RBucket, + hash [32]byte, primaryPaymentID int64, sqlDB SQLQueries, + stats *MigrationStats) error { + + duplicateCount := 0 + + err := dupBucket.ForEach(func(seqBytes, _ []byte) error { + // The duplicates bucket should only contain nested buckets + // keyed by 8-byte sequence numbers. Skip any unexpected keys + // (defensive check for corrupted or malformed data). + if len(seqBytes) != 8 { + log.Warnf("Skipping unexpected key in duplicates "+ + "bucket for payment %x: key length %d, "+ + "expected 8", + hash[:8], len(seqBytes), + ) + + return nil + } + + seqNum := byteOrder.Uint64(seqBytes) + subBucket := dupBucket.NestedReadBucket(seqBytes) + if subBucket == nil { + return nil + } + + duplicateCount++ + log.Infof("Migrating duplicate payment seq=%d for "+ + "payment %x", seqNum, hash[:8]) + + err := migrateSingleDuplicatePayment( + ctx, subBucket, hash, primaryPaymentID, seqNum, + sqlDB, + ) + if err != nil { + return fmt.Errorf( + "migrate duplicate payment seq=%d: %w", + seqNum, err, + ) + } + + return nil + }) + + if duplicateCount > 0 { + stats.DuplicatePayments++ + stats.DuplicateEntries += int64(duplicateCount) + + log.Infof("Payment %x had %d duplicate(s) migrated", hash[:8], + duplicateCount) + } + + return err +} + +// migrateSingleDuplicatePayment inserts a duplicate payment record for the +// given payment hash into payment_duplicates. +func migrateSingleDuplicatePayment(ctx context.Context, dupBucket kvdb.RBucket, + hash [32]byte, primaryPaymentID int64, duplicateSeq uint64, + sqlDB SQLQueries) error { + + creationData := dupBucket.Get(duplicatePaymentCreationInfoKey) + if creationData == nil { + return fmt.Errorf("duplicate payment seq=%d missing "+ + "creation info (payment=%x)", duplicateSeq, hash[:8]) + } + + creationInfo, err := deserializeDuplicatePaymentCreationInfo( + bytes.NewReader(creationData), + ) + if err != nil { + return fmt.Errorf("deserialize duplicate creation "+ + "info: %w", err) + } + + settleData := dupBucket.Get(duplicatePaymentSettleInfoKey) + failReasonData := dupBucket.Get(duplicatePaymentFailInfoKey) + attemptData := dupBucket.Get(duplicatePaymentAttemptInfoKey) + + if settleData != nil && len(failReasonData) > 0 { + return fmt.Errorf("duplicate payment seq=%d has both "+ + "settle and fail info (payment=%x)", duplicateSeq, + hash[:8]) + } + + var ( + failReason sql.NullInt32 + settlePreimage []byte + settleTime sql.NullTime + ) + + switch { + case settleData != nil: + settlePreimage, settleTime, err = parseDuplicateSettleData( + settleData, + ) + if err != nil { + return err + } + + case len(failReasonData) > 0: + failReason = sql.NullInt32{ + Int32: int32(failReasonData[0]), + Valid: true, + } + + default: + // If the duplicate payment has no settle or fail info, + // we mark it as failed during the migration. Duplicate + // payments were a bug in older versions of LND, so we can be + // sure if a duplicate payment has no failure reason or + // settlement data, the corresponding HTLC for this payment + // has been failed (resolved). + if attemptData == nil { + log.Warnf("Duplicate payment seq=%d has no "+ + "attempt info and no resolution (payment=%x); "+ + "marking failed", duplicateSeq, hash[:8]) + } else { + log.Warnf("Duplicate payment seq=%d has attempt "+ + "info but no resolution (payment=%x); "+ + "marking failed", duplicateSeq, hash[:8]) + } + + failReason = sql.NullInt32{ + Int32: int32(FailureReasonError), + Valid: true, + } + } + + _, err = sqlDB.InsertPaymentDuplicateMig( + ctx, sqlc.InsertPaymentDuplicateMigParams{ + PaymentID: primaryPaymentID, + AmountMsat: int64(creationInfo.Value), + CreatedAt: normalizeTimeForSQL( + creationInfo.CreationTime, + ), + FailReason: failReason, + SettlePreimage: settlePreimage, + SettleTime: settleTime, + }, + ) + if err != nil { + return fmt.Errorf("insert duplicate payment: %w", err) + } + + return nil +} + +// parseDuplicateSettleData extracts settle data from either legacy or modern +// duplicate formats. +func parseDuplicateSettleData(settleData []byte) ([]byte, sql.NullTime, error) { + if len(settleData) == lntypes.PreimageSize { + return append([]byte(nil), settleData...), sql.NullTime{}, nil + } + + settleInfo, err := deserializeHTLCSettleInfo( + bytes.NewReader(settleData), + ) + if err != nil { + return nil, sql.NullTime{}, + fmt.Errorf("deserialize duplicate settle: %w", err) + } + + settleTime := normalizeTimeForSQL(settleInfo.SettleTime) + + return settleInfo.Preimage[:], sql.NullTime{ + Time: settleTime, + Valid: !settleTime.IsZero(), + }, nil +} + +// printMigrationSummary prints a summary of the migration. +func printMigrationSummary(stats *MigrationStats) { + if stats.TotalPayments == 0 { + log.Infof("No payments migrated - database is empty") + + return + } + + log.Infof("========================================") + log.Infof(" Payment Migration Summary") + log.Infof("========================================") + log.Infof("Total Payments: %d", stats.TotalPayments) + log.Infof(" Successful: %d", stats.SuccessfulPayments) + log.Infof(" Failed: %d", stats.FailedPayments) + log.Infof(" In-Flight: %d", stats.InFlightPayments) + log.Infof(" Initiated: %d", stats.InitiatedPayments) + log.Infof("") + log.Infof("Total HTLC Attempts: %d", stats.TotalAttempts) + log.Infof(" Settled: %d", stats.SettledAttempts) + log.Infof(" Failed: %d", stats.FailedAttempts) + log.Infof(" In-Flight: %d", stats.InFlightAttempts) + log.Infof("") + log.Infof("Total Route Hops: %d", stats.TotalHops) + + if stats.SkippedPayments > 0 { + log.Infof("") + log.Warnf("SKIPPED PAYMENTS:") + log.Warnf(" Indexed payments with missing buckets: %d", + stats.SkippedPayments) + log.Warnf(" These indicate minor DB inconsistencies.") + } + + if stats.DuplicatePayments > 0 { + log.Infof("") + log.Warnf("DUPLICATE PAYMENTS DETECTED:") + log.Warnf(" Unique payment hashes with duplicates: %d", + stats.DuplicatePayments) + log.Warnf(" Total duplicate entries migrated: %d", + stats.DuplicateEntries) + log.Warnf(" These were caused by an old LND bug.") + } + + log.Infof("") + log.Infof("Migration Duration: %v", stats.MigrationDuration) + log.Infof("========================================") +} From f058f3329d1e9e290e2d5b235e8cecbeedd9210f Mon Sep 17 00:00:00 2001 From: ziggie Date: Sun, 11 Jan 2026 00:05:29 +0100 Subject: [PATCH 05/16] payments/migration1: add migration test suite and helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add test helpers plus sql_migration_test coverage for KV→SQL migration. Basic migration, sequence ordering, data integrity, and feature-specific cases (MPP/AMP, custom records, blinded routes, metadata, failure messages). Also cover duplicate payment migration to payment_duplicates, including missing attempt info to ensure terminal failure is recorded. This gives broad regression coverage for the migration path and its edge-cases. --- payments/db/migration1/sql_migration_test.go | 2622 ++++++++++++++++++ payments/db/migration1/test_harness.go | 26 + payments/db/migration1/test_postgres.go | 94 + payments/db/migration1/test_sql.go | 58 + payments/db/migration1/test_sqlite.go | 73 + 5 files changed, 2873 insertions(+) create mode 100644 payments/db/migration1/sql_migration_test.go create mode 100644 payments/db/migration1/test_harness.go create mode 100644 payments/db/migration1/test_postgres.go create mode 100644 payments/db/migration1/test_sql.go create mode 100644 payments/db/migration1/test_sqlite.go diff --git a/payments/db/migration1/sql_migration_test.go b/payments/db/migration1/sql_migration_test.go new file mode 100644 index 000000000..f1e96f025 --- /dev/null +++ b/payments/db/migration1/sql_migration_test.go @@ -0,0 +1,2622 @@ +//go:build test_db_postgres || test_db_sqlite + +package migration1 + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "sort" + "testing" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// TestMigrationKVToSQL tests the basic payment migration from KV to SQL. +func TestMigrationKVToSQL(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // Setup KV database and populate with test data. + kvDB := setupTestKVDB(t) + populateTestPayments(t, kvDB, 5) + + sqlStore := setupTestSQLDB(t) + + // Run migration in a single transaction. + err := runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) +} + +// TestMigrationSequenceOrder ensures the migration follows sequence order +// rather than lexicographic hash order. +func TestMigrationSequenceOrder(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + kvDB := setupTestKVDB(t) + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + var globalAttemptID uint64 + hash0 := [32]byte{} + hash1 := [32]byte{} + hash2 := [32]byte{} + hash0[0] = 3 + hash1[0] = 2 + hash2[0] = 1 + + if err := createTestPaymentInKV( + t, paymentsBucket, indexBucket, 0, hash0, + &globalAttemptID, + ); err != nil { + return err + } + + // We make sure that the duplicate payment is skipped because + // it will be migrated separately into payment_duplicates. + if err := createTestDuplicatePaymentWithIndex( + t, paymentsBucket, indexBucket, hash0, 1, false, + &globalAttemptID, + ); err != nil { + return err + } + if err := createTestPaymentInKV( + t, paymentsBucket, indexBucket, 2, hash1, + &globalAttemptID, + ); err != nil { + return err + } + if err := createTestPaymentInKV( + t, paymentsBucket, indexBucket, 3, hash2, + &globalAttemptID, + ); err != nil { + return err + } + + return nil + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + resp, err := sqlStore.QueryPayments(ctx, Query{ + MaxPayments: 10, + IncludeIncomplete: true, + }) + require.NoError(t, err) + require.Len(t, resp.Payments, 3) + + var ( + exp0 lntypes.Hash + exp1 lntypes.Hash + exp2 lntypes.Hash + ) + exp0[0] = 3 + exp1[0] = 2 + exp2[0] = 1 + + require.Equal(t, exp0, resp.Payments[0].Info.PaymentIdentifier) + require.Equal(t, exp1, resp.Payments[1].Info.PaymentIdentifier) + require.Equal(t, exp2, resp.Payments[2].Info.PaymentIdentifier) +} + +// TestMigrationDataIntegrity verifies that migrated payment data exactly +// matches the original KV data when fetched through the SQLStore +// (SQLStore.FetchPayment). This covers the SQLStore query path separately +// from the migration's own batch validation. +func TestMigrationDataIntegrity(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // Setup KV database with test data. + kvDB := setupTestKVDB(t) + numPayments := populateTestPayments(t, kvDB, 5) + + // Fetch all payments from KV before migration. + kvPayments := fetchAllPaymentsFromKV(t, kvDB) + require.Len(t, kvPayments, numPayments) + + // Setup SQL database and run migration. + sqlStore := setupTestSQLDB(t) + + err := runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Compare each KV payment with its SQL counterpart using deep equality. + // This ensures that ALL fields match, not just a few selected ones. + for _, kvPayment := range kvPayments { + comparePaymentData(t, ctx, sqlStore, kvPayment) + } +} + +// TestMigrationWithDuplicates tests migration of duplicate payments into +// the payment_duplicates table. +func TestMigrationWithDuplicates(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // Setup KV database. + kvDB := setupTestKVDB(t) + + // Create a payment with duplicates. + hash := createTestPaymentHash(t, 0) + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + // Create root buckets. + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + // Create primary payment with sequence 0 and globally unique + // attempt ID. + var globalAttemptID uint64 + err = createTestPaymentInKV( + t, paymentsBucket, indexBucket, 0, hash, + &globalAttemptID, + ) + if err != nil { + return err + } + + // Add 2 duplicate payments for the same hash. + paymentBucket := paymentsBucket.NestedReadWriteBucket(hash[:]) + require.NotNil(t, paymentBucket) + + dupBucket, err := paymentBucket.CreateBucketIfNotExists( + duplicatePaymentsBucket, + ) + if err != nil { + return err + } + + // Create duplicate with sequence 1 using global attempt ID. + err = createTestDuplicatePayment( + t, dupBucket, hash, 1, true, &globalAttemptID, + ) + if err != nil { + return err + } + + // Create duplicate with sequence 2 using global attempt ID. + err = createTestDuplicatePayment( + t, dupBucket, hash, 2, false, &globalAttemptID, + ) + if err != nil { + return err + } + + return nil + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + // Run migration. + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Verify in SQL database. + var count int64 + err = sqlStore.db.ExecTx( + ctx, sqldb.ReadTxOpt(), func(q SQLQueries) error { + var err error + count, err = q.CountPayments(ctx) + return err + }, sqldb.NoOpReset, + ) + require.NoError(t, err) + require.Equal( + t, int64(1), count, "SQL DB should have 1 payment", + ) + + var ( + dbPayment sqlc.FetchPaymentRow + duplicates []sqlc.PaymentDuplicate + ) + err = sqlStore.db.ExecTx( + ctx, sqldb.ReadTxOpt(), func(q SQLQueries) error { + var err error + dbPayment, err = q.FetchPayment(ctx, hash[:]) + if err != nil { + return err + } + + duplicates, err = q.FetchPaymentDuplicates( + ctx, dbPayment.Payment.ID, + ) + return err + }, sqldb.NoOpReset, + ) + require.NoError(t, err) + + require.Len(t, duplicates, 2) + sort.SliceStable(duplicates, func(i, j int) bool { + return duplicates[i].AmountMsat < duplicates[j].AmountMsat + }) + + require.Equal(t, int64(2001000), duplicates[0].AmountMsat) + require.False(t, duplicates[0].FailReason.Valid) + require.NotEmpty(t, duplicates[0].SettlePreimage) + + require.Equal(t, int64(2002000), duplicates[1].AmountMsat) + require.True(t, duplicates[1].FailReason.Valid) + require.Equal( + t, int32(FailureReasonError), + duplicates[1].FailReason.Int32, + ) + require.Empty(t, duplicates[1].SettlePreimage) +} + +// TestDuplicatePaymentsWithoutAttemptInfo verifies duplicate payments without +// attempt info are migrated with terminal failure reasons. +func TestDuplicatePaymentsWithoutAttemptInfo(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + hash := createTestPaymentHash(t, 0) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + var globalAttemptID uint64 + err = createTestPaymentInKV( + t, paymentsBucket, indexBucket, 1, hash, + &globalAttemptID, + ) + if err != nil { + return err + } + + paymentBucket := paymentsBucket.NestedReadWriteBucket( + hash[:], + ) + require.NotNil(t, paymentBucket) + + dupBucket, err := paymentBucket.CreateBucketIfNotExists( + duplicatePaymentsBucket, + ) + if err != nil { + return err + } + + if err := createDuplicateWithoutAttemptInfo( + t, dupBucket, hash, 2, true, false, + ); err != nil { + return err + } + if err := createDuplicateWithoutAttemptInfo( + t, dupBucket, hash, 3, false, true, + ); err != nil { + return err + } + if err := createDuplicateWithoutAttemptInfo( + t, dupBucket, hash, 4, false, false, + ); err != nil { + return err + } + + return nil + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + var ( + dbPayment sqlc.FetchPaymentRow + duplicates []sqlc.PaymentDuplicate + ) + err = sqlStore.db.ExecTx( + ctx, sqldb.ReadTxOpt(), func(q SQLQueries) error { + var err error + dbPayment, err = q.FetchPayment(ctx, hash[:]) + if err != nil { + return err + } + + duplicates, err = q.FetchPaymentDuplicates( + ctx, dbPayment.Payment.ID, + ) + return err + }, sqldb.NoOpReset, + ) + require.NoError(t, err) + + require.Len(t, duplicates, 3) + sort.SliceStable(duplicates, func(i, j int) bool { + return duplicates[i].AmountMsat < duplicates[j].AmountMsat + }) + + require.Equal(t, int64(2002000), duplicates[0].AmountMsat) + require.NotEmpty(t, duplicates[0].SettlePreimage) + require.False(t, duplicates[0].FailReason.Valid) + + require.Equal(t, int64(2003000), duplicates[1].AmountMsat) + require.True(t, duplicates[1].FailReason.Valid) + require.Equal( + t, int32(FailureReasonNoRoute), + duplicates[1].FailReason.Int32, + ) + require.Empty(t, duplicates[1].SettlePreimage) + + require.Equal(t, int64(2004000), duplicates[2].AmountMsat) + require.True(t, duplicates[2].FailReason.Valid) + require.Equal( + t, int32(FailureReasonError), + duplicates[2].FailReason.Int32, + ) + require.Empty(t, duplicates[2].SettlePreimage) +} + +// TestMigratePaymentWithMPP tests migration of a payment with MPP (multi-path +// payment) records. +func TestMigratePaymentWithMPP(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + // Create a payment with MPP. + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_mpp_payment_hash_12345")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithMPP( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + // Run migration. + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Verify payment matches. + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithAMP tests migration of a payment with AMP (atomic +// multi-path) records. +func TestMigratePaymentWithAMP(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_amp_payment_hash_12345")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithAMP( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithAMPSignedChildIndex tests migration of an AMP payment +// where the child index has the signed bit set. +func TestMigratePaymentWithAMPSignedChildIndex(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_amp_child_idx_8000")) + + const childIndex = uint32(0x80000001) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithAMPChildIndex( + t, paymentsBucket, indexBucket, paymentHash, childIndex, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithCustomRecords tests migration of a payment with custom +// records. +func TestMigratePaymentWithCustomRecords(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_custom_records_hash_12")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithCustomRecords( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithBlindedRoute tests migration of a payment with blinded +// route. +func TestMigratePaymentWithBlindedRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_blinded_route_hash_123")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithBlindedRoute( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithMetadata tests migration of a payment with hop +// metadata. +func TestMigratePaymentWithMetadata(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_metadata_payment_hash_")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithMetadata( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithAllFeatures tests migration with all optional +// features enabled. +func TestMigratePaymentWithAllFeatures(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_all_features_hash_1234")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithAllFeatures( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentFeatureCombinations tests selected feature combinations +// in a single migration to cover interactions without random data. +func TestMigratePaymentFeatureCombinations(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + cases := []paymentFeatureSet{ + { + name: "mpp_custom", + mpp: true, + customRecords: true, + }, + { + name: "amp_blinded", + amp: true, + blindedRoute: true, + }, + { + name: "custom_metadata", + customRecords: true, + hopMetadata: true, + }, + { + name: "blinded_metadata", + blindedRoute: true, + hopMetadata: true, + }, + { + name: "mpp_metadata", + mpp: true, + hopMetadata: true, + }, + } + + hashes := make([][32]byte, 0, len(cases)) + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + var globalAttemptID uint64 + for i, c := range cases { + hash := sha256.Sum256([]byte(c.name)) + hashes = append(hashes, hash) + + err := createPaymentWithFeatureSet( + t, paymentsBucket, indexBucket, hash, + uint64(10+i), c, &globalAttemptID, + ) + if err != nil { + return err + } + } + + return nil + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + for _, hash := range hashes { + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, hash) + } +} + +// TestMigratePaymentWithFailureMessage tests migration of a payment with a +// failed HTLC that includes a failure message. +func TestMigratePaymentWithFailureMessage(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_fail_msg_hash_123456789")) + + // Create a payment with a failed HTLC. + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + // Create payment bucket. + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists( + paymentHash[:], + ) + if err != nil { + return err + } + + // Add creation info. + var paymentID lntypes.Hash + copy(paymentID[:], paymentHash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(1000000), + CreationTime: time.Now().Add(-24 * time.Hour), + PaymentRequest: []byte("lnbc10utest"), + } + + // Use a separate buffer for payment creation info to avoid + // reuse issues when serializing HTLC attempts later. + var creationInfoBuf bytes.Buffer + err = serializePaymentCreationInfo( + &creationInfoBuf, creationInfo, + ) + if err != nil { + return err + } + + serialized := creationInfoBuf.Bytes() + + err = paymentBucket.Put( + paymentCreationInfoKey, serialized, + ) + if err != nil { + return err + } + + // Add sequence number. + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, 50) + err = paymentBucket.Put(paymentSequenceKey, seqBytes) + if err != nil { + return err + } + + // Add payment-level failure reason. + failReasonBytes := []byte{byte(FailureReasonNoRoute)} + err = paymentBucket.Put( + paymentFailInfoKey, failReasonBytes, + ) + if err != nil { + return err + } + + // Create HTLC bucket with one failed attempt. + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + // Create the failed attempt with a failure message. + attemptID := uint64(500) + sessionKey, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + hopKey, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + // Create a proper copy of the hash instead of referencing + // the local variable directly. + attemptHash := new(lntypes.Hash) + copy(attemptHash[:], paymentHash[:]) + + //nolint:ll + attemptInfo := &HTLCAttemptInfo{ + AttemptID: attemptID, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: 500000, + TotalAmount: 900, + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex(hopKey.PubKey()), + ChannelID: 12345, + OutgoingTimeLock: 499500, + AmtToForward: 850, + }, + }, + }, + AttemptTime: time.Now().Add(-2 * time.Hour), + Hash: attemptHash, + } + + // Write attempt info. + attemptKey := make([]byte, len(htlcAttemptInfoKey)+8) + copy(attemptKey, htlcAttemptInfoKey) + byteOrder.PutUint64( + attemptKey[len(htlcAttemptInfoKey):], attemptID, + ) + + var b bytes.Buffer + err = serializeHTLCAttemptInfo(&b, attemptInfo) + if err != nil { + return err + } + err = htlcBucket.Put(attemptKey, b.Bytes()) + if err != nil { + return err + } + + // Add failure info with a message. + //nolint:ll + failInfo := &HTLCFailInfo{ + FailTime: time.Now().Add(-1 * time.Hour), + Message: &lnwire.FailTemporaryChannelFailure{}, + Reason: HTLCFailMessage, + FailureSourceIndex: 1, + } + + failKey := make([]byte, len(htlcFailInfoKey)+8) + copy(failKey, htlcFailInfoKey) + byteOrder.PutUint64(failKey[len(htlcFailInfoKey):], attemptID) + + b.Reset() + if err := serializeHTLCFailInfo(&b, failInfo); err != nil { + return err + } + if err := htlcBucket.Put(failKey, b.Bytes()); err != nil { + return err + } + + // Create index entry. + var idx bytes.Buffer + if err := WriteElements( + &idx, paymentIndexTypeHash, paymentHash[:], + ); err != nil { + return err + } + return indexBucket.Put(seqBytes, idx.Bytes()) + }, func() {}) + require.NoError(t, err) + + // Migrate to SQL. + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Verify data matches. + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// setupTestKVDB creates a temporary KV database for testing. +func setupTestKVDB(t *testing.T) kvdb.Backend { + t.Helper() + + backend, cleanup, err := kvdb.GetTestBackend(t.TempDir(), "payments") + require.NoError(t, err) + t.Cleanup(cleanup) + + return backend +} + +// populateTestPayments populates the KV database with test payment data. +func populateTestPayments(t *testing.T, db kvdb.Backend, numPayments int) int { + t.Helper() + + err := kvdb.Update(db, func(tx kvdb.RwTx) error { + // Create root buckets. + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + // Create test payments with globally unique attempt IDs. + var globalAttemptID uint64 + for i := 0; i < numPayments; i++ { + hash := createTestPaymentHash(t, i) + + err := createTestPaymentInKV( + t, paymentsBucket, indexBucket, uint64(i), hash, + &globalAttemptID, + ) + if err != nil { + return err + } + } + + return nil + }, func() {}) + + require.NoError(t, err) + return numPayments +} + +// serializeDuplicatePaymentCreationInfo serializes PaymentCreationInfo for +// duplicate payments. The time is stored in seconds (not nanoseconds) to match +// the format used by deserializeDuplicatePaymentCreationInfo in the KV store. +func serializeDuplicatePaymentCreationInfo(w io.Writer, + c *PaymentCreationInfo) error { + + var scratch [8]byte + + if _, err := w.Write(c.PaymentIdentifier[:]); err != nil { + return err + } + + byteOrder.PutUint64(scratch[:], uint64(c.Value)) + if _, err := w.Write(scratch[:]); err != nil { + return err + } + + // Store time in seconds (not nanoseconds) for duplicate payments. + // This matches the deserialization format used in + // deserializeDuplicatePaymentCreationInfo. + var unixSec int64 + if !c.CreationTime.IsZero() { + unixSec = c.CreationTime.Unix() + } + byteOrder.PutUint64(scratch[:], uint64(unixSec)) + if _, err := w.Write(scratch[:]); err != nil { + return err + } + + byteOrder.PutUint32(scratch[:4], uint32(len(c.PaymentRequest))) + if _, err := w.Write(scratch[:4]); err != nil { + return err + } + + if _, err := w.Write(c.PaymentRequest); err != nil { + return err + } + + return nil +} + +// createTestPaymentHash creates a deterministic payment hash for testing. +func createTestPaymentHash(t *testing.T, seed int) [32]byte { + t.Helper() + + hash := sha256.Sum256([]byte{byte(seed)}) + return hash +} + +// createTestPaymentInKV creates a single payment in the KV store. +func createTestPaymentInKV(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, seqNum uint64, hash [32]byte, + globalAttemptID *uint64) error { + + t.Helper() + + // Create payment bucket. + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists(hash[:]) + if err != nil { + return err + } + + // Create payment creation info. + var paymentID lntypes.Hash + copy(paymentID[:], hash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(1000000 + seqNum*1000), + CreationTime: time.Now().Add(-24 * time.Hour), + PaymentRequest: []byte("lnbc1test"), + } + + // Serialize and write creation info. + var b bytes.Buffer + err = serializePaymentCreationInfo(&b, creationInfo) + if err != nil { + return err + } + err = paymentBucket.Put(paymentCreationInfoKey, b.Bytes()) + if err != nil { + return err + } + + // Store sequence number. + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, seqNum) + err = paymentBucket.Put(paymentSequenceKey, seqBytes) + if err != nil { + return err + } + + // Add one HTLC attempt for each payment with globally unique ID. + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + // Increment global attempt ID and create HTLC attempt. So we have a + // globally unique attempt ID for the HTLC attempt. + *globalAttemptID++ + err = createTestHTLCAttempt( + t, htlcBucket, hash, *globalAttemptID, seqNum%3 == 0, + ) + if err != nil { + return err + } + + var idx bytes.Buffer + err = WriteElements(&idx, paymentIndexTypeHash, hash[:]) + if err != nil { + return err + } + + return indexBucket.Put(seqBytes, idx.Bytes()) +} + +// createTestHTLCAttempt creates a test HTLC attempt in the KV store. +func createTestHTLCAttempt(t *testing.T, htlcBucket kvdb.RwBucket, + paymentHash [32]byte, attemptID uint64, shouldSettle bool) error { + t.Helper() + + // Generate a session key. + sessionKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + // Create a simple 2-hop route. + hop1Key, err := btcec.NewPrivateKey() + require.NoError(t, err) + + hop2Key, err := btcec.NewPrivateKey() + require.NoError(t, err) + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + // Convert session key to [32]byte. + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + attemptInfo := &HTLCAttemptInfo{ + AttemptID: attemptID, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: 500000, + TotalAmount: 900, + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex( + hop1Key.PubKey(), + ), + ChannelID: 12345, + OutgoingTimeLock: 499500, + AmtToForward: 850, + }, + { + PubKeyBytes: NewVertex( + hop2Key.PubKey(), + ), + ChannelID: 67890, + OutgoingTimeLock: 499000, + AmtToForward: 800, + }, + }, + }, + AttemptTime: time.Now().Add(-2 * time.Hour), + Hash: (*lntypes.Hash)(&paymentHash), + } + + // Serialize and write attempt info. + attemptKey := make([]byte, len(htlcAttemptInfoKey)+8) + copy(attemptKey, htlcAttemptInfoKey) + byteOrder.PutUint64(attemptKey[len(htlcAttemptInfoKey):], attemptID) + + var b bytes.Buffer + err = serializeHTLCAttemptInfo(&b, attemptInfo) + if err != nil { + return err + } + err = htlcBucket.Put(attemptKey, b.Bytes()) + if err != nil { + return err + } + + // Add settlement if requested. + if shouldSettle { + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(paymentHash), + SettleTime: time.Now().Add(-1 * time.Hour), + } + + settleKey := make([]byte, len(htlcSettleInfoKey)+8) + copy(settleKey, htlcSettleInfoKey) + byteOrder.PutUint64( + settleKey[len(htlcSettleInfoKey):], attemptID, + ) + + var sb bytes.Buffer + err = serializeHTLCSettleInfo(&sb, settleInfo) + if err != nil { + return err + } + err = htlcBucket.Put(settleKey, sb.Bytes()) + if err != nil { + return err + } + } + + return nil +} + +// createTestDuplicatePaymentWithIndex creates a duplicate payment and adds +// a matching entry into the global payment sequence index. +func createTestDuplicatePaymentWithIndex(t *testing.T, + paymentsBucket kvdb.RwBucket, indexBucket kvdb.RwBucket, + paymentHash [32]byte, seqNum uint64, shouldSettle bool, + globalAttemptID *uint64) error { + t.Helper() + + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists( + paymentHash[:], + ) + if err != nil { + return err + } + + dupBucket, err := paymentBucket.CreateBucketIfNotExists( + duplicatePaymentsBucket, + ) + if err != nil { + return err + } + + if err := createTestDuplicatePayment( + t, dupBucket, paymentHash, seqNum, shouldSettle, + globalAttemptID, + ); err != nil { + return err + } + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, seqNum) + var idx bytes.Buffer + if err := WriteElements( + &idx, paymentIndexTypeHash, paymentHash[:], + ); err != nil { + return err + } + + return indexBucket.Put(seqBytes, idx.Bytes()) +} + +// createTestDuplicatePayment creates a duplicate payment in the KV store. +func createTestDuplicatePayment(t *testing.T, + dupBucket kvdb.RwBucket, paymentHash [32]byte, seqNum uint64, + shouldSettle bool, globalAttemptID *uint64) error { + + t.Helper() + + // Create bucket for this duplicate using sequence number as key. + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, seqNum) + + dupPaymentBucket, err := dupBucket.CreateBucketIfNotExists(seqBytes) + if err != nil { + return err + } + + // Store sequence number. + err = dupPaymentBucket.Put(duplicatePaymentSequenceKey, seqBytes) + if err != nil { + return err + } + + // Create payment creation info. + var paymentID lntypes.Hash + copy(paymentID[:], paymentHash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(2000000 + seqNum*1000), + CreationTime: time.Now().Add(-48 * time.Hour), + PaymentRequest: []byte("lnbc1duplicate"), + } + + var b bytes.Buffer + err = serializeDuplicatePaymentCreationInfo(&b, creationInfo) + if err != nil { + return err + } + err = dupPaymentBucket.Put(duplicatePaymentCreationInfoKey, b.Bytes()) + if err != nil { + return err + } + + // Generate a session key for the duplicate attempt. + sessionKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + // Create route for duplicate. + hop1Key, err := btcec.NewPrivateKey() + require.NoError(t, err) + + hop2Key, err := btcec.NewPrivateKey() + require.NoError(t, err) + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + // Use globally unique attempt ID. + *globalAttemptID++ + attemptID := *globalAttemptID + + duplicateAttempt := &duplicateHTLCAttemptInfo{ + attemptID: attemptID, + sessionKey: sessionKeyBytes, + route: Route{ + TotalTimeLock: 500000, + TotalAmount: 900, + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex( + hop1Key.PubKey(), + ), + ChannelID: 12345, + OutgoingTimeLock: 499500, + AmtToForward: 850, + }, + { + PubKeyBytes: NewVertex( + hop2Key.PubKey(), + ), + ChannelID: 67890, + OutgoingTimeLock: 499000, + AmtToForward: 800, + }, + }, + }, + } + + // Serialize and write attempt info (using existing WriteElements + // and SerializeRoute). + var ab bytes.Buffer + if err := WriteElements( + &ab, duplicateAttempt.attemptID, + duplicateAttempt.sessionKey, + ); err != nil { + return err + } + if err := SerializeRoute(&ab, duplicateAttempt.route); err != nil { + return err + } + err = dupPaymentBucket.Put(duplicatePaymentAttemptInfoKey, ab.Bytes()) + if err != nil { + return err + } + + // Add settlement if requested. + if shouldSettle { + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(paymentHash), + SettleTime: time.Now().Add(-1 * time.Hour), + } + + var sb bytes.Buffer + err = serializeHTLCSettleInfo(&sb, settleInfo) + if err != nil { + return err + } + err = dupPaymentBucket.Put( + duplicatePaymentSettleInfoKey, sb.Bytes(), + ) + if err != nil { + return err + } + } + + return nil +} + +// createDuplicateWithoutAttemptInfo creates a duplicate payment bucket with +// settle/fail info but without attempt info. +func createDuplicateWithoutAttemptInfo(t *testing.T, + dupBucket kvdb.RwBucket, paymentHash [32]byte, seqNum uint64, + shouldSettle bool, shouldFail bool) error { + t.Helper() + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, seqNum) + + dupPaymentBucket, err := dupBucket.CreateBucketIfNotExists(seqBytes) + if err != nil { + return err + } + + if err := dupPaymentBucket.Put( + duplicatePaymentSequenceKey, seqBytes, + ); err != nil { + return err + } + + var paymentID lntypes.Hash + copy(paymentID[:], paymentHash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(2000000 + seqNum*1000), + CreationTime: time.Now().Add(-48 * time.Hour), + PaymentRequest: []byte("lnbc1duplicate"), + } + + var b bytes.Buffer + err = serializeDuplicatePaymentCreationInfo(&b, creationInfo) + if err != nil { + return err + } + if err := dupPaymentBucket.Put( + duplicatePaymentCreationInfoKey, b.Bytes(), + ); err != nil { + return err + } + + switch { + case shouldSettle && shouldFail: + return fmt.Errorf("invalid duplicate state") + case shouldSettle: + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(paymentHash), + SettleTime: time.Now().Add(-1 * time.Hour), + } + + var sb bytes.Buffer + err = serializeHTLCSettleInfo(&sb, settleInfo) + if err != nil { + return err + } + if err := dupPaymentBucket.Put( + duplicatePaymentSettleInfoKey, sb.Bytes(), + ); err != nil { + return err + } + case shouldFail: + failReasonBytes := []byte{byte(FailureReasonNoRoute)} + if err := dupPaymentBucket.Put( + duplicatePaymentFailInfoKey, failReasonBytes, + ); err != nil { + return err + } + } + + return nil +} + +// fetchAllPaymentsFromKV fetches all payments from the KV store using the +// KVStore implementation. +func fetchAllPaymentsFromKV(t *testing.T, kvDB kvdb.Backend) []*MPPayment { + t.Helper() + + kvStore, err := NewKVStore(kvDB, WithNoMigration(true)) + require.NoError(t, err) + + payments, err := kvStore.FetchPayments() + require.NoError(t, err) + + return payments +} + +// normalizePaymentData makes sure that the payment data is normalized for +// comparison using the same logic as in-migration validation. +func normalizePaymentData(payment *MPPayment) { + normalizePaymentForCompare(payment) +} + +// comparePaymentData compares a KV payment with its SQL counterpart using +// deep equality check (similar to invoice migration). +func comparePaymentData(t *testing.T, ctx context.Context, sqlStore *SQLStore, + kvPayment *MPPayment) { + + t.Helper() + + // Fetch the SQL payment as MPPayment using SQLStore. + var paymentHash lntypes.Hash + copy(paymentHash[:], kvPayment.Info.PaymentIdentifier[:]) + + sqlPayment, err := sqlStore.FetchPayment(ctx, paymentHash) + require.NoError(t, err, "SQL payment should exist for %x", + paymentHash[:8]) + + // Normalize time precision to microseconds. + normalizePaymentData(kvPayment) + normalizePaymentData(sqlPayment) + + // Deep equality check - compares all fields recursively. + require.Equal(t, kvPayment, sqlPayment, + "KV and SQL payments should be equal for %x", paymentHash[:8]) +} + +// runPaymentsMigration executes the payment migration from KV to SQL within +// a SQL transaction. +func runPaymentsMigration(ctx context.Context, kvDB kvdb.Backend, + sqlStore *SQLStore) error { + + return sqlStore.db.ExecTx( + ctx, sqldb.WriteTxOpt(), func(tx SQLQueries) error { + migTx, ok := tx.(SQLMigrationQueries) + if !ok { + return fmt.Errorf("db does not implement " + + "SQLMigrationQueries") + } + + return MigratePaymentsKVToSQL( + ctx, kvDB, migTx, &SQLStoreConfig{ + QueryCfg: sqlStore.cfg.QueryCfg, + }, + ) + }, sqldb.NoOpReset, + ) +} + +// assertPaymentDataMatches verifies a payment in KV matches its SQL counterpart +// using deep equality check. +func assertPaymentDataMatches(t *testing.T, ctx context.Context, + kvDB kvdb.Backend, sqlStore *SQLStore, hash [32]byte) { + t.Helper() + + // Fetch from KV. + var kvPayment *MPPayment + err := kvdb.View(kvDB, func(tx kvdb.RTx) error { + paymentsBucket := tx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + return nil + } + + paymentBucket := paymentsBucket.NestedReadBucket(hash[:]) + if paymentBucket == nil { + return nil + } + + var err error + kvPayment, err = fetchPayment(paymentBucket) + return err + }, func() {}) + require.NoError(t, err) + + if kvPayment == nil { + // Payment doesn't exist in KV, should not exist in SQL + // either. + var paymentHash lntypes.Hash + copy(paymentHash[:], hash[:]) + _, err := sqlStore.FetchPayment(ctx, paymentHash) + require.Error( + t, err, "payment should not exist in SQL if not "+ + "in KV", + ) + return + } + + // Use the deep comparison function. + comparePaymentData(t, ctx, sqlStore, kvPayment) +} + +// paymentTestConfig holds configuration for creating test payments with various +// features. +type paymentTestConfig struct { + hash [32]byte + seqNum uint64 + value lnwire.MilliSatoshi + creationTime time.Time + paymentRequest string + attemptID uint64 + numHops int + baseChannelID uint64 + baseTimeLock uint32 + paymentCustomRecs lnwire.CustomRecords + attemptCustomRecs lnwire.CustomRecords + hopConfigurator func(hop *Hop, index int, isFinal bool) +} + +// serializeAndPut serializes data using the provided serializer function and +// writes it to the bucket. +func serializeAndPut(bucket kvdb.RwBucket, key []byte, + serializer func(io.Writer) error) error { + + var b bytes.Buffer + if err := serializer(&b); err != nil { + return err + } + return bucket.Put(key, b.Bytes()) +} + +// generateSessionKey creates a new session key and returns the private key, +// source public key vertex, and serialized key bytes. +func generateSessionKey(t *testing.T) (*btcec.PrivateKey, Vertex, + [32]byte, error) { + + t.Helper() + + sessionKey, err := btcec.NewPrivateKey() + if err != nil { + return nil, Vertex{}, [32]byte{}, err + } + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + return sessionKey, sourcePubKey, sessionKeyBytes, nil +} + +// createTestHops creates the specified number of test hops with the given +// parameters. The configurator function is called for each hop to allow +// feature-specific customization. +func createTestHops(t *testing.T, numHops int, baseAmount lnwire.MilliSatoshi, + baseChannelID uint64, baseTimeLock uint32, + configurator func(*Hop, int, bool)) ([]*Hop, + lnwire.MilliSatoshi, error) { + + t.Helper() + + hops := make([]*Hop, numHops) + currentAmt := baseAmount + + for i := 0; i < numHops; i++ { + hopKey, err := btcec.NewPrivateKey() + if err != nil { + return nil, 0, err + } + + amt := baseAmount - lnwire.MilliSatoshi(uint64(i)*100) + hop := &Hop{ + PubKeyBytes: NewVertex(hopKey.PubKey()), + ChannelID: baseChannelID + uint64(i), + OutgoingTimeLock: baseTimeLock - uint32(i*40), + AmtToForward: amt, + } + + // Apply feature-specific configuration. + if configurator != nil { + configurator(hop, i, i == numHops-1) + } + + hops[i] = hop + currentAmt = amt + } + + return hops, currentAmt, nil +} + +// writeHTLCAttempt writes the HTLC attempt info to the bucket. +func writeHTLCAttempt(bucket kvdb.RwBucket, attemptID uint64, + info *HTLCAttemptInfo) error { + + attemptKey := make([]byte, len(htlcAttemptInfoKey)+8) + copy(attemptKey, htlcAttemptInfoKey) + byteOrder.PutUint64(attemptKey[len(htlcAttemptInfoKey):], attemptID) + + return serializeAndPut(bucket, attemptKey, func(w io.Writer) error { + return serializeHTLCAttemptInfo(w, info) + }) +} + +// writeHTLCSettle writes the HTLC settle info to the bucket. +func writeHTLCSettle(bucket kvdb.RwBucket, attemptID uint64, + info *HTLCSettleInfo) error { + + settleKey := make([]byte, len(htlcSettleInfoKey)+8) + copy(settleKey, htlcSettleInfoKey) + byteOrder.PutUint64(settleKey[len(htlcSettleInfoKey):], attemptID) + + return serializeAndPut(bucket, settleKey, func(w io.Writer) error { + return serializeHTLCSettleInfo(w, info) + }) +} + +// createIndexEntry creates a payment index entry in the index bucket. +func createIndexEntry(indexBucket kvdb.RwBucket, seqBytes []byte, + hash [32]byte) error { + + var idx bytes.Buffer + if err := WriteElements( + &idx, paymentIndexTypeHash, hash[:], + ); err != nil { + return err + } + + return indexBucket.Put(seqBytes, idx.Bytes()) +} + +// createTestPayment creates a test payment with the specified configuration, +// handling all common boilerplate code. +func createTestPayment(t *testing.T, paymentsBucket, indexBucket kvdb.RwBucket, + cfg paymentTestConfig) error { + + t.Helper() + + // Create payment bucket. + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists( + cfg.hash[:], + ) + if err != nil { + return err + } + + // Create payment ID. + var paymentID lntypes.Hash + copy(paymentID[:], cfg.hash[:]) + + // Create and serialize payment creation info. + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: cfg.value, + CreationTime: cfg.creationTime, + PaymentRequest: []byte(cfg.paymentRequest), + FirstHopCustomRecords: cfg.paymentCustomRecs, + } + + err = serializeAndPut( + paymentBucket, paymentCreationInfoKey, + func(w io.Writer) error { + return serializePaymentCreationInfo(w, creationInfo) + }, + ) + if err != nil { + return err + } + + // Store sequence number. + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, cfg.seqNum) + if err := paymentBucket.Put(paymentSequenceKey, seqBytes); err != nil { + return err + } + + // Create HTLC bucket. + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + // Generate session key. + _, sourcePubKey, sessionKeyBytes, err := generateSessionKey(t) + if err != nil { + return err + } + + // Create route with hops. + hops, totalAmount, err := createTestHops( + t, cfg.numHops, cfg.value, cfg.baseChannelID, + cfg.baseTimeLock, cfg.hopConfigurator, + ) + if err != nil { + return err + } + + // Create and serialize attempt info. + attemptInfo := &HTLCAttemptInfo{ + AttemptID: cfg.attemptID, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: cfg.baseTimeLock, + TotalAmount: totalAmount, + SourcePubKey: sourcePubKey, + Hops: hops, + FirstHopWireCustomRecords: cfg.attemptCustomRecs, + }, + AttemptTime: cfg.creationTime.Add(time.Minute), + Hash: (*lntypes.Hash)(&cfg.hash), + } + + if err = writeHTLCAttempt( + htlcBucket, cfg.attemptID, attemptInfo, + ); err != nil { + return err + } + + // Add settlement. + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(cfg.hash), + SettleTime: cfg.creationTime.Add(2 * time.Minute), + } + + if err := writeHTLCSettle( + htlcBucket, cfg.attemptID, settleInfo, + ); err != nil { + return err + } + + // Create index entry. + return createIndexEntry(indexBucket, seqBytes, cfg.hash) +} + +// createPaymentWithMPP creates a payment with MPP records on the final hop. +func createPaymentWithMPP(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 1, + value: 50000, + creationTime: time.Date( + 2024, 1, 1, 12, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc500n1test_mpp", + attemptID: 1, + numHops: 3, + baseChannelID: 100000, + baseTimeLock: 500000, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + if isFinal { + var paymentAddr [32]byte + copy( + paymentAddr[:], + []byte( + "test_mpp_payment_"+ + "address_32", + ), + ) + hop.MPP = record.NewMPP( + lnwire.MilliSatoshi(50000), + paymentAddr, + ) + } + }, + }, + ) +} + +// createPaymentWithAMP creates a payment with AMP records on the final hop. +func createPaymentWithAMP(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + t.Helper() + + return createPaymentWithAMPChildIndex( + t, paymentsBucket, indexBucket, hash, 0, + ) +} + +// createPaymentWithAMPChildIndex creates a payment with AMP records on the +// final hop and a specific child index. +func createPaymentWithAMPChildIndex(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte, childIndex uint32) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 2, + value: 75000, + creationTime: time.Date( + 2024, 2, 1, 10, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc750n1test_amp", + attemptID: 1, + numHops: 2, + baseChannelID: 200000, + baseTimeLock: 600000, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + if isFinal { + var rootShare [32]byte + copy( + rootShare[:], + []byte( + "test_amp_root_share"+ + "_12345678", + ), + ) + var setID [32]byte + copy( + setID[:], + []byte( + "test_amp_set_id_"+ + "123456789012", + ), + ) + hop.AMP = record.NewAMP( + rootShare, setID, childIndex, + ) + } + }, + }, + ) +} + +// createPaymentWithCustomRecords creates a payment with custom records at all +// levels. +func createPaymentWithCustomRecords(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 3, + value: 100000, + creationTime: time.Date( + 2024, 3, 1, 14, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc1m1test_custom", + attemptID: 1, + numHops: 3, + baseChannelID: 300000, + baseTimeLock: 700000, + paymentCustomRecs: lnwire.CustomRecords{ + 65536: []byte("payment_level_value_1"), + 65537: []byte("payment_level_value_2"), + }, + attemptCustomRecs: lnwire.CustomRecords{ + 65541: []byte("attempt_custom_value_1"), + 65542: []byte("attempt_custom_value_2"), + }, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + hop.CustomRecords = record.CustomSet{ + 65538 + uint64(index): []byte( + fmt.Sprintf( + "hop_%d_custom_value", + index, + ), + ), + } + }, + }, + ) +} + +// createPaymentWithBlindedRoute creates a payment with blinded route data. +func createPaymentWithBlindedRoute(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 4, + value: 120000, + creationTime: time.Date( + 2024, 4, 1, 16, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc1200n1test_blinded", + attemptID: 1, + numHops: 4, + baseChannelID: 400000, + baseTimeLock: 800000, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + if isFinal { + blindingKey, err := btcec. + NewPrivateKey() + + require.NoError(t, err) + + hop.BlindingPoint = blindingKey.PubKey() + hop.EncryptedData = []byte( + "encrypted_blinded_route_" + + "data_test_value_12345", + ) + hop.TotalAmtMsat = lnwire.MilliSatoshi( + 119400, + ) + } + }, + }, + ) +} + +// createPaymentWithMetadata creates a payment with hop metadata. +func createPaymentWithMetadata(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 5, + value: 80000, + creationTime: time.Date( + 2024, 5, 1, 18, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc800n1test_metadata", + attemptID: 1, + numHops: 3, + baseChannelID: 500000, + baseTimeLock: 900000, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + hop.Metadata = []byte( + fmt.Sprintf( + "hop_%d_metadata_value", index, + ), + ) + }, + }, + ) +} + +// createPaymentWithAllFeatures creates a payment with all optional features +// enabled. +func createPaymentWithAllFeatures(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 6, + value: 150000, + creationTime: time.Date( + 2024, 6, 1, 20, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc1500n1test_all_features", + attemptID: 1, + numHops: 4, + baseChannelID: 600000, + baseTimeLock: 1000000, + paymentCustomRecs: lnwire.CustomRecords{ + 65543: []byte("all_features_payment_custom_1"), + 65544: []byte("all_features_payment_custom_2"), + }, + attemptCustomRecs: lnwire.CustomRecords{ + 65549: []byte("all_feat_attempt_custom_1"), + 65550: []byte("all_feat_attempt_custom_2"), + }, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + // Add custom records and metadata to all hops. + hop.CustomRecords = record.CustomSet{ + 65545 + uint64(index): []byte( + fmt.Sprintf( + "all_feat_hop_%d", + index, + ), + ), + } + hop.Metadata = []byte( + fmt.Sprintf( + "all_feat_metadata_%d", index, + ), + ) + + // Add MPP and blinded route data to final hop. + if isFinal { + var paymentAddr [32]byte + copy( + paymentAddr[:], + []byte( + "all_features_mpp_"+ + "addr_123456", + ), + ) + hop.MPP = record.NewMPP( + lnwire.MilliSatoshi(149250), + paymentAddr, + ) + + blindingKey, err := btcec. + NewPrivateKey() + + require.NoError(t, err) + hop.BlindingPoint = blindingKey.PubKey() + hop.EncryptedData = []byte( + "all_features_encrypted_" + + "blinded_data_123456", + ) + hop.TotalAmtMsat = lnwire.MilliSatoshi( + 149250, + ) + } + }, + }, + ) +} + +// paymentFeatureSet defines a combination of optional payment features for +// testing feature interactions. +type paymentFeatureSet struct { + name string + mpp bool + amp bool + customRecords bool + blindedRoute bool + hopMetadata bool +} + +// TestMigrateLegacyPaymentWithNilHash tests migration of a legacy payment where +// the HTLC attempt's Hash field is nil. In older payments, the Hash wasn't +// stored on individual HTLCs; instead, the parent payment hash should be used. +func TestMigrateLegacyPaymentWithNilHash(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_legacy_nil_hash_payment")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createLegacyPaymentWithNilHash( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Verify the payment was migrated correctly. + var paymentID lntypes.Hash + copy(paymentID[:], paymentHash[:]) + + payment, err := sqlStore.FetchPayment(ctx, paymentID) + require.NoError(t, err) + require.NotNil(t, payment) + require.Len(t, payment.HTLCs, 1) + + // The HTLC should have the parent payment hash since its own Hash was + // nil. + require.NotNil(t, payment.HTLCs[0].Hash) + require.Equal(t, paymentID, *payment.HTLCs[0].Hash) +} + +// createLegacyPaymentWithNilHash creates a payment with an HTLC attempt that +// has a nil Hash field, simulating older payment data format. +func createLegacyPaymentWithNilHash(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists(hash[:]) + if err != nil { + return err + } + + var paymentID lntypes.Hash + copy(paymentID[:], hash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(50000), + CreationTime: time.Now().Add(-24 * time.Hour), + PaymentRequest: []byte("lnbc500n1legacy_payment"), + } + + var creationBuf bytes.Buffer + err = serializePaymentCreationInfo(&creationBuf, creationInfo) + if err != nil { + return err + } + err = paymentBucket.Put(paymentCreationInfoKey, creationBuf.Bytes()) + if err != nil { + return err + } + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, 100) + err = paymentBucket.Put(paymentSequenceKey, seqBytes) + if err != nil { + return err + } + + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + // Create HTLC attempt with nil Hash (legacy format). + sessionKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + hopKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + attemptInfo := &HTLCAttemptInfo{ + AttemptID: 1, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: 500000, + TotalAmount: 50000, + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex(hopKey.PubKey()), + ChannelID: 12345, + OutgoingTimeLock: 499500, + AmtToForward: 49000, + }, + }, + }, + AttemptTime: time.Now().Add(-2 * time.Hour), + Hash: nil, // Legacy: Hash is nil + } + + // Serialize the attempt info. Since Hash is nil, the serialization + // will not write the hash, simulating the legacy format. + attemptKey := make([]byte, len(htlcAttemptInfoKey)+8) + copy(attemptKey, htlcAttemptInfoKey) + byteOrder.PutUint64(attemptKey[len(htlcAttemptInfoKey):], 1) + + var attemptBuf bytes.Buffer + err = serializeHTLCAttemptInfo(&attemptBuf, attemptInfo) + if err != nil { + return err + } + err = htlcBucket.Put(attemptKey, attemptBuf.Bytes()) + if err != nil { + return err + } + + // Add settlement info. + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(hash), + SettleTime: time.Now().Add(-1 * time.Hour), + } + + settleKey := make([]byte, len(htlcSettleInfoKey)+8) + copy(settleKey, htlcSettleInfoKey) + byteOrder.PutUint64(settleKey[len(htlcSettleInfoKey):], 1) + + var settleBuf bytes.Buffer + err = serializeHTLCSettleInfo(&settleBuf, settleInfo) + if err != nil { + return err + } + err = htlcBucket.Put(settleKey, settleBuf.Bytes()) + if err != nil { + return err + } + + // Create index entry. + var idx bytes.Buffer + err = WriteElements(&idx, paymentIndexTypeHash, hash[:]) + if err != nil { + return err + } + + return indexBucket.Put(seqBytes, idx.Bytes()) +} + +// createPaymentWithFeatureSet creates a payment with a selected set of +// optional features for combination testing. +func createPaymentWithFeatureSet(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte, seqNum uint64, + features paymentFeatureSet, globalAttemptID *uint64) error { + t.Helper() + + if features.mpp && features.amp { + return fmt.Errorf("invalid feature set: mpp and amp") + } + + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists(hash[:]) + if err != nil { + return err + } + + var paymentID lntypes.Hash + copy(paymentID[:], hash[:]) + + creationTime := time.Date(2024, 7, 1, 12, 0, 0, 0, time.UTC). + Add(time.Duration(seqNum) * time.Minute) + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(100000), + CreationTime: creationTime, + PaymentRequest: []byte( + fmt.Sprintf("lnbc_test_%s", features.name), + ), + } + if features.customRecords { + creationInfo.FirstHopCustomRecords = lnwire.CustomRecords{ + 65560: []byte("combo_payment_custom_1"), + 65561: []byte("combo_payment_custom_2"), + } + } + + var b bytes.Buffer + err = serializePaymentCreationInfo(&b, creationInfo) + if err != nil { + return err + } + err = paymentBucket.Put(paymentCreationInfoKey, b.Bytes()) + if err != nil { + return err + } + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, seqNum) + err = paymentBucket.Put(paymentSequenceKey, seqBytes) + if err != nil { + return err + } + + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + sessionKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + baseAmt := lnwire.MilliSatoshi(100000) + hops := make([]*Hop, 3) + for i := 0; i < 3; i++ { + hopKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + amt := baseAmt - lnwire.MilliSatoshi(uint64(i)*100) + hop := &Hop{ + PubKeyBytes: NewVertex(hopKey.PubKey()), + ChannelID: uint64(700000 + i), + OutgoingTimeLock: uint32(700000 - i*40), + AmtToForward: amt, + } + if features.customRecords { + hop.CustomRecords = record.CustomSet{ + 65562 + uint64(i): []byte(fmt.Sprintf( + "combo_hop_%d", i, + )), + } + } + if features.hopMetadata { + hop.Metadata = []byte( + fmt.Sprintf("combo_metadata_%d", i), + ) + } + + if i == 2 { + if features.mpp { + var paymentAddr [32]byte + copy( + paymentAddr[:], + []byte("combo_mpp_payment_addr_1234"), + ) + hop.MPP = record.NewMPP( + baseAmt-200, paymentAddr, + ) + } + if features.amp { + var rootShare [32]byte + copy( + rootShare[:], + []byte("combo_amp_root_share_123456"), + ) + var setID [32]byte + copy( + setID[:], + []byte("combo_amp_set_id_12345678"), + ) + hop.AMP = record.NewAMP(rootShare, setID, 0) + } + if features.blindedRoute { + blindingKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + hop.BlindingPoint = blindingKey.PubKey() + hop.EncryptedData = []byte( + "combo_encrypted_blinded_data", + ) + hop.TotalAmtMsat = baseAmt - 200 + } + } + + hops[i] = hop + } + + routeInfo := Route{ + TotalTimeLock: 700000, + TotalAmount: baseAmt - 200, + SourcePubKey: sourcePubKey, + Hops: hops, + } + if features.customRecords { + routeInfo.FirstHopWireCustomRecords = lnwire.CustomRecords{ + 65565: []byte("combo_attempt_custom_1"), + 65566: []byte("combo_attempt_custom_2"), + } + } + + *globalAttemptID++ + attemptID := *globalAttemptID + attemptInfo := &HTLCAttemptInfo{ + AttemptID: attemptID, + sessionKey: sessionKeyBytes, + Route: routeInfo, + AttemptTime: creationTime.Add(time.Minute), + Hash: (*lntypes.Hash)(&hash), + } + + attemptKey := make([]byte, len(htlcAttemptInfoKey)+8) + copy(attemptKey, htlcAttemptInfoKey) + byteOrder.PutUint64(attemptKey[len(htlcAttemptInfoKey):], attemptID) + + var ab bytes.Buffer + err = serializeHTLCAttemptInfo(&ab, attemptInfo) + if err != nil { + return err + } + err = htlcBucket.Put(attemptKey, ab.Bytes()) + if err != nil { + return err + } + + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(hash), + SettleTime: creationTime.Add(2 * time.Minute), + } + + settleKey := make([]byte, len(htlcSettleInfoKey)+8) + copy(settleKey, htlcSettleInfoKey) + byteOrder.PutUint64(settleKey[len(htlcSettleInfoKey):], attemptID) + + var sb bytes.Buffer + err = serializeHTLCSettleInfo(&sb, settleInfo) + if err != nil { + return err + } + err = htlcBucket.Put(settleKey, sb.Bytes()) + if err != nil { + return err + } + + var idx bytes.Buffer + err = WriteElements(&idx, paymentIndexTypeHash, hash[:]) + if err != nil { + return err + } + + return indexBucket.Put(seqBytes, idx.Bytes()) +} + +// TestMigrateInFlightPayment tests that a payment with an active (in-flight) +// HTLC attempt — i.e. a node that upgrades while a payment is still pending on +// the network — is migrated correctly. The HTLC attempt must appear in SQL +// without a settlement or failure resolution, and FetchInFlightPayments must +// return the payment after migration. +func TestMigrateInFlightPayment(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_inflight_payment_hash_1234")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket( + paymentsIndexBucket, + ) + if err != nil { + return err + } + + return createInFlightPayment( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Verify the payment data matches between KV and SQL. + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) + + // Additionally verify that FetchInFlightPayments returns this payment, + // confirming it is treated as in-flight by the SQL store. + inFlight, err := sqlStore.FetchInFlightPayments(ctx) + require.NoError(t, err) + require.Len(t, inFlight, 1) + + var paymentID lntypes.Hash + copy(paymentID[:], paymentHash[:]) + require.Equal(t, paymentID, inFlight[0].Info.PaymentIdentifier) + require.Equal(t, StatusInFlight, inFlight[0].Status) + require.Len(t, inFlight[0].HTLCs, 1) + require.Nil(t, inFlight[0].HTLCs[0].Settle) + require.Nil(t, inFlight[0].HTLCs[0].Failure) +} + +// createInFlightPayment creates a payment with an active (in-flight) HTLC +// attempt in the KV store. Unlike settled payments, no settle or fail +// resolution is written for the HTLC, simulating a payment that is still +// pending on the network at the time of migration. +func createInFlightPayment(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists(hash[:]) + if err != nil { + return err + } + + var paymentID lntypes.Hash + copy(paymentID[:], hash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(100000), + CreationTime: time.Now().Add(-1 * time.Hour), + PaymentRequest: []byte("lnbc1inflight_payment"), + } + + err = serializeAndPut( + paymentBucket, paymentCreationInfoKey, + func(w io.Writer) error { + return serializePaymentCreationInfo(w, creationInfo) + }, + ) + if err != nil { + return err + } + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, 42) + if err := paymentBucket.Put(paymentSequenceKey, seqBytes); err != nil { + return err + } + + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + _, sourcePubKey, sessionKeyBytes, err := generateSessionKey(t) + if err != nil { + return err + } + + hopKey, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + const attemptID = 1 + attemptInfo := &HTLCAttemptInfo{ + AttemptID: attemptID, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: 500000, + TotalAmount: 100000, + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex( + hopKey.PubKey(), + ), + ChannelID: 12345, + OutgoingTimeLock: 499500, + AmtToForward: 99000, + }, + }, + }, + AttemptTime: time.Now().Add(-30 * time.Minute), + Hash: (*lntypes.Hash)(&hash), + } + + if err = writeHTLCAttempt(htlcBucket, attemptID, attemptInfo); err != nil { + return err + } + + return createIndexEntry(indexBucket, seqBytes, hash) +} diff --git a/payments/db/migration1/test_harness.go b/payments/db/migration1/test_harness.go new file mode 100644 index 000000000..b25867a4f --- /dev/null +++ b/payments/db/migration1/test_harness.go @@ -0,0 +1,26 @@ +package migration1 + +import ( + "testing" + + "github.com/lightningnetwork/lnd/lntypes" +) + +// TestHarness provides implementation-specific test utilities for the payments +// database. Different database backends (KV, SQL) have different internal +// structures and indexing mechanisms, so this interface allows tests to verify +// implementation-specific behavior without coupling the test logic to a +// particular backend. +type TestHarness interface { + // AssertPaymentIndex checks that a payment is correctly indexed. + // For KV: verifies the payment index bucket entry exists and points + // to the correct payment hash. + // For SQL: no-op (SQL doesn't use a separate index bucket). + AssertPaymentIndex(t *testing.T, expectedHash lntypes.Hash) + + // AssertNoIndex checks that an index for a sequence number doesn't + // exist. + // For KV: verifies the index bucket entry is deleted. + // For SQL: no-op. + AssertNoIndex(t *testing.T, seqNr uint64) +} diff --git a/payments/db/migration1/test_postgres.go b/payments/db/migration1/test_postgres.go new file mode 100644 index 000000000..7055fb885 --- /dev/null +++ b/payments/db/migration1/test_postgres.go @@ -0,0 +1,94 @@ +//go:build test_db_postgres && !test_db_sqlite + +package migration1 + +import ( + "testing" + + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// NewTestDB is a helper function that creates a SQLStore backed by a SQL +// database for testing. +func NewTestDB(t testing.TB, opts ...OptionModifier) (DB, TestHarness) { + db := NewTestDBWithFixture(t, nil, opts...) + return db, &noopTestHarness{} +} + +// NewTestDBFixture creates a new sqldb.TestPgFixture for testing purposes. +func NewTestDBFixture(t *testing.T) *sqldb.TestPgFixture { + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + return pgFixture +} + +// NewTestDBWithFixture is a helper function that creates a SQLStore backed by a +// SQL database for testing. +func NewTestDBWithFixture(t testing.TB, + pgFixture *sqldb.TestPgFixture, opts ...OptionModifier) DB { + + var querier BatchedSQLQueries + if pgFixture == nil { + querier = newBatchQuerier(t) + } else { + querier = newBatchQuerierWithFixture(t, pgFixture) + } + + store, err := NewSQLStore( + &SQLStoreConfig{ + QueryCfg: sqldb.DefaultPostgresConfig(), + }, querier, opts..., + ) + require.NoError(t, err) + + return store +} + +// newBatchQuerier creates a new BatchedSQLQueries instance for testing +// using a PostgreSQL database fixture. +func newBatchQuerier(t testing.TB) BatchedSQLQueries { + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + + return newBatchQuerierWithFixture(t, pgFixture) +} + +// newBatchQuerierWithFixture creates a new BatchedSQLQueries instance for +// testing using a PostgreSQL database fixture. +func newBatchQuerierWithFixture(t testing.TB, + pgFixture *sqldb.TestPgFixture) BatchedSQLQueries { + + rawDB := sqldb.NewTestPostgresDB(t, pgFixture).BaseDB.DB + + return &testBatchedSQLQueries{ + db: rawDB, + Queries: sqlc.New(rawDB), + } +} + +// noopTestHarness is the SQL test harness implementation. Since SQL doesn't +// use a separate payment index bucket like KV, these assertions are no-ops. +type noopTestHarness struct{} + +// AssertPaymentIndex is a no-op for SQL implementations. +func (h *noopTestHarness) AssertPaymentIndex(t *testing.T, + expectedHash lntypes.Hash) { + + // No-op: SQL doesn't use a separate index bucket. +} + +// AssertNoIndex is a no-op for SQL implementations. +func (h *noopTestHarness) AssertNoIndex(t *testing.T, seqNr uint64) { + // No-op: SQL doesn't use a separate index bucket. +} diff --git a/payments/db/migration1/test_sql.go b/payments/db/migration1/test_sql.go new file mode 100644 index 000000000..f2d4d078b --- /dev/null +++ b/payments/db/migration1/test_sql.go @@ -0,0 +1,58 @@ +//go:build test_db_postgres || test_db_sqlite + +package migration1 + +import ( + "context" + "database/sql" + "testing" + + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// setupTestSQLDB creates a SQLStore-backed test database. +func setupTestSQLDB(t testing.TB, opts ...OptionModifier) *SQLStore { + t.Helper() + + db, _ := NewTestDB(t, opts...) + sqlStore, ok := db.(*SQLStore) + require.True(t, ok) + + return sqlStore +} + +// testBatchedSQLQueries is a simple implementation of BatchedSQLQueries for +// testing. +type testBatchedSQLQueries struct { + db *sql.DB + *sqlc.Queries +} + +// ExecTx implements the transaction execution logic. +func (t *testBatchedSQLQueries) ExecTx(ctx context.Context, + txOpts sqldb.TxOptions, txBody func(SQLQueries) error, + reset func()) error { + + sqlOptions := sql.TxOptions{ + Isolation: sql.LevelSerializable, + ReadOnly: txOpts.ReadOnly(), + } + + tx, err := t.db.BeginTx(ctx, &sqlOptions) + if err != nil { + return err + } + + reset() + queries := sqlc.New(tx) + + if err := txBody(queries); err != nil { + _ = tx.Rollback() + + return err + } + + return tx.Commit() +} diff --git a/payments/db/migration1/test_sqlite.go b/payments/db/migration1/test_sqlite.go new file mode 100644 index 000000000..b84c9c2d9 --- /dev/null +++ b/payments/db/migration1/test_sqlite.go @@ -0,0 +1,73 @@ +//go:build !test_db_postgres && test_db_sqlite + +package migration1 + +import ( + "testing" + + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// NewTestDB is a helper function that creates a SQLStore backed by a SQL +// database for testing. +func NewTestDB(t testing.TB, opts ...OptionModifier) (DB, TestHarness) { + db := NewTestDBWithFixture(t, nil, opts...) + return db, &noopTestHarness{} +} + +// NewTestDBFixture is a no-op for the sqlite build. +func NewTestDBFixture(_ *testing.T) *sqldb.TestPgFixture { + return nil +} + +// NewTestDBWithFixture is a helper function that creates a SQLStore backed by a +// SQL database for testing. +func NewTestDBWithFixture(t testing.TB, _ *sqldb.TestPgFixture, + opts ...OptionModifier) DB { + + store, err := NewSQLStore( + &SQLStoreConfig{ + QueryCfg: sqldb.DefaultSQLiteConfig(), + }, newBatchQuerier(t), opts..., + ) + require.NoError(t, err) + return store +} + +// newBatchQuerier creates a new BatchedSQLQueries instance for testing +// using a SQLite database. +func newBatchQuerier(t testing.TB) BatchedSQLQueries { + return newBatchQuerierWithFixture(t, nil) +} + +// newBatchQuerierWithFixture creates a new BatchedSQLQueries instance for +// testing using a SQLite database. +func newBatchQuerierWithFixture(t testing.TB, + _ *sqldb.TestPgFixture) BatchedSQLQueries { + + rawDB := sqldb.NewTestSqliteDB(t).BaseDB.DB + + return &testBatchedSQLQueries{ + db: rawDB, + Queries: sqlc.New(rawDB), + } +} + +// noopTestHarness is the SQL test harness implementation. Since SQL doesn't +// use a separate payment index bucket like KV, these assertions are no-ops. +type noopTestHarness struct{} + +// AssertPaymentIndex is a no-op for SQL implementations. +func (h *noopTestHarness) AssertPaymentIndex(t *testing.T, + expectedHash lntypes.Hash) { + + // No-op: SQL doesn't use a separate index bucket. +} + +// AssertNoIndex is a no-op for SQL implementations. +func (h *noopTestHarness) AssertNoIndex(t *testing.T, seqNr uint64) { + // No-op: SQL doesn't use a separate index bucket. +} From 4b30bed0ac957b3526e36faf75e523614b20eb2c Mon Sep 17 00:00:00 2001 From: ziggie Date: Sun, 11 Jan 2026 00:07:36 +0100 Subject: [PATCH 06/16] payments/migration1: add external migration test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a developer-facing migration_external_test that allows running the KV→SQL payments migration against a real channel.db backend to debug migration failures on actual data. The accompanying testdata README documents how to supply a database file and configure the test, so users can validate their data and confirm the migration completes successfully. The test is skipped by default and meant for manual diagnostics. --- .../db/migration1/migration_external_test.go | 367 ++++++++++++++++++ payments/db/migration1/testdata/README.md | 52 +++ 2 files changed, 419 insertions(+) create mode 100644 payments/db/migration1/migration_external_test.go create mode 100644 payments/db/migration1/testdata/README.md diff --git a/payments/db/migration1/migration_external_test.go b/payments/db/migration1/migration_external_test.go new file mode 100644 index 000000000..a5ddc42c8 --- /dev/null +++ b/payments/db/migration1/migration_external_test.go @@ -0,0 +1,367 @@ +//go:build test_db_postgres || test_db_sqlite + +package migration1 + +import ( + "context" + "fmt" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/btcsuite/btclog/v2" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/kvdb/postgres" + "github.com/lightningnetwork/lnd/kvdb/sqlbase" + "github.com/lightningnetwork/lnd/kvdb/sqlite" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// TestMigrationWithExternalDB tests the migration of the payment store from a +// bolt backed channel.db or a kvdb channel.sqlite to a SQL database. Note that +// this test does not attempt to be a complete migration test for all payment +// store types but rather is added as a tool for developers and users to debug +// payment migration issues with an actual channel.db/channel.sqlite file. +// +// NOTE: To use this test, place either of those files in the +// payments/db/migration1/testdata directory, uncomment the "Skipf" line, and +// set the "fileName" variable to the name of the channel database file you +// want to use for the migration test. +func TestMigrationWithExternalDB(t *testing.T) { + ctx := context.Background() + + // NOTE: comment this line out to run the test. + t.Skipf("skipping test meant for local debugging only") + + // NOTE: set this to the name of the channel database file you want + // to use for the migration test. This may be either a bbolt ".db" file + // or a SQLite ".sqlite" file. If you want to migrate from a + // bbolt channel.db file, set this to "channel.db". + const fileName = "channel.db" + + // NOTE: if set, this test will prefer migrating from a Postgres-backed + // kvdb source instead of a local file. Leave empty to use fileName. + const postgresKVDSN = "" + const postgresKVPfx = "channeldb" + const logSequenceOrder = false + + // Determine if we are using a SQLite file or a Bolt DB file. + isSqlite := strings.HasSuffix(fileName, ".sqlite") + + // Set up logging for the test. + logger := btclog.NewSLogger(btclog.NewDefaultHandler(os.Stdout)) + UseLogger(logger) + + // migrate runs the migration from the kvdb store to the SQL store + // and then performs a batched deep comparison of every migrated + // payment to ensure all data (including HTLC details) was preserved + // correctly. + migrate := func(t *testing.T, kvBackend kvdb.Backend) { + sqlStore := setupTestSQLDB(t) + + // Run migration in a transaction. + err := sqlStore.db.ExecTx( + ctx, sqldb.WriteTxOpt(), func(tx SQLQueries) error { + return MigratePaymentsKVToSQL( + ctx, kvBackend, tx, &SQLStoreConfig{ + QueryCfg: sqlStore.cfg.QueryCfg, + }, + ) + }, sqldb.NoOpReset, + ) + require.NoError(t, err) + + t.Log("========================================") + t.Log(" Deep Validation") + t.Log("========================================") + + // Deep compare all migrated payments in batches. + deepValidateAllPayments( + t, ctx, kvBackend, sqlStore, + ) + + _ = logSequenceOrder + } + + connectPostgres := func(t *testing.T, dsn, prefix string) kvdb.Backend { + dsn = strings.TrimSpace(dsn) + if dsn == "" { + t.Fatalf("missing postgres kvdb dsn") + } + + prefix = strings.TrimSpace(prefix) + if prefix == "" { + prefix = "channeldb" + } + + const ( + timeout = 10 * time.Second + maxConns = 5 + ) + sqlbase.Init(maxConns) + + dbCfg := &postgres.Config{ + Dsn: dsn, + Timeout: timeout, + MaxConnections: maxConns, + } + + kvStore, err := kvdb.Open( + kvdb.PostgresBackendName, ctx, dbCfg, prefix, + ) + require.NoError(t, err) + + t.Cleanup(func() { _ = kvStore.Close() }) + + return kvStore + } + + connectPostgresKV := func(t *testing.T) kvdb.Backend { + return connectPostgres(t, postgresKVDSN, postgresKVPfx) + } + + connectBBolt := func(t *testing.T, dbPath string) kvdb.Backend { + cfg := &kvdb.BoltBackendConfig{ + DBPath: dbPath, + DBFileName: fileName, + NoFreelistSync: true, + AutoCompact: false, + AutoCompactMinAge: kvdb.DefaultBoltAutoCompactMinAge, + DBTimeout: kvdb.DefaultDBTimeout, + ReadOnly: true, + } + + kvStore, err := kvdb.GetBoltBackend(cfg) + require.NoError(t, err) + + t.Cleanup(func() { _ = kvStore.Close() }) + + return kvStore + } + + connectSQLite := func(t *testing.T, dbPath string) kvdb.Backend { + const ( + timeout = 10 * time.Second + maxConns = 5 + ) + sqlbase.Init(maxConns) + + cfg := &sqlite.Config{ + Timeout: timeout, + BusyTimeout: timeout, + MaxConnections: maxConns, + } + + kvStore, err := kvdb.Open( + kvdb.SqliteBackendName, ctx, cfg, + dbPath, fileName, + // NOTE: we use the raw string here else we get an + // import cycle if we try to import lncfg.NSChannelDB. + "channeldb", + ) + require.NoError(t, err) + + t.Cleanup(func() { _ = kvStore.Close() }) + + return kvStore + } + + tests := []struct { + name string + dbPath string + }{ + { + name: "testdata", + dbPath: "testdata", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if postgresKVDSN != "" { + migrate(t, connectPostgresKV(t)) + return + } + + chanDBPath := path.Join(test.dbPath, fileName) + t.Logf("Connecting to channel DB at: %s", chanDBPath) + + connectDB := connectBBolt + if isSqlite { + connectDB = connectSQLite + } + + migrate(t, connectDB(t, test.dbPath)) + }) + } +} + +// kvPaymentRef holds a KV payment and its hash for batched deep validation. +type kvPaymentRef struct { + hash lntypes.Hash + payment *MPPayment +} + +// deepValidateAllPayments iterates all KV payments in batches and performs a +// deep comparison against their SQL counterparts. For each batch, it fetches +// the full SQL payment data (including HTLCs, routes, custom records) via +// batch queries and compares field-by-field with the KV data. +func deepValidateAllPayments(t *testing.T, ctx context.Context, + kvBackend kvdb.Backend, sqlStore *SQLStore) { + + t.Helper() + + batchSize := int(sqlStore.cfg.QueryCfg.MaxBatchSize) + + // Get total payment count from SQL so we can show progress. + var totalPayments int64 + err := sqlStore.db.ExecTx( + ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + var err error + totalPayments, err = db.CountPayments(ctx) + return err + }, sqldb.NoOpReset, + ) + require.NoError(t, err) + + var totalValidated int + + // Iterate all KV payments, collecting them into batches. For each + // full batch, perform a batched deep comparison against SQL. + var batch []kvPaymentRef + err = kvdb.View(kvBackend, func(kvTx kvdb.RTx) error { + paymentsBucket := kvTx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + return nil + } + + return paymentsBucket.ForEach(func(k, v []byte) error { + bucket := paymentsBucket.NestedReadBucket(k) + if bucket == nil { + return nil + } + + payment, err := fetchPayment(bucket) + if err != nil { + return err + } + + var hash lntypes.Hash + copy(hash[:], k) + + batch = append(batch, kvPaymentRef{ + hash: hash, + payment: payment, + }) + + if len(batch) >= batchSize { + deepCompareBatch( + t, ctx, sqlStore, batch, + ) + totalValidated += len(batch) + t.Logf("Deep validated %d/%d payments", + totalValidated, totalPayments) + + batch = batch[:0] + } + + return nil + }) + }, func() { + batch = nil + totalValidated = 0 + }) + require.NoError(t, err) + + // Validate any remaining payments in the last batch. + if len(batch) > 0 { + deepCompareBatch(t, ctx, sqlStore, batch) + totalValidated += len(batch) + } + + t.Logf("Deep validated %d/%d payments (complete)", + totalValidated, totalPayments) +} + +// deepCompareBatch performs a batched deep comparison of KV payments against +// their SQL counterparts. It fetches SQL payment data in a single read +// transaction using batch queries. +func deepCompareBatch(t *testing.T, ctx context.Context, + sqlStore *SQLStore, batch []kvPaymentRef) { + + t.Helper() + + err := sqlStore.db.ExecTx( + ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + // Look up each payment by hash to get the SQL row + // with payment ID. + ids := make([]int64, 0, len(batch)) + rowsByHash := make( + map[lntypes.Hash]sqlc.FetchPaymentsByIDsRow, + len(batch), + ) + for _, ref := range batch { + row, err := fetchPaymentByHash( + ctx, db, ref.hash, + ) + if err != nil { + return fmt.Errorf("fetch SQL payment "+ + "%x: %w", ref.hash[:8], err) + } + ids = append(ids, row.Payment.ID) + } + + // Batch-fetch full payment details. + rows, err := db.FetchPaymentsByIDs(ctx, ids) + if err != nil { + return fmt.Errorf("batch fetch payments: %w", + err) + } + for _, row := range rows { + var hash lntypes.Hash + copy(hash[:], row.PaymentIdentifier) + rowsByHash[hash] = row + } + + batchData, err := batchLoadPaymentDetailsData( + ctx, sqlStore.cfg.QueryCfg, db, ids, + ) + if err != nil { + return fmt.Errorf("batch load details: %w", + err) + } + + // Deep compare each payment. + for _, ref := range batch { + row, ok := rowsByHash[ref.hash] + require.True(t, ok, + "SQL payment %x not in batch "+ + "results", ref.hash[:8]) + + sqlPayment, err := buildPaymentFromBatchData( + row, batchData, + ) + require.NoError(t, err, + "build SQL payment %x", + ref.hash[:8]) + + normalizePaymentData(ref.payment) + normalizePaymentData(sqlPayment) + + require.Equal( + t, ref.payment, sqlPayment, + "payment mismatch %x", + ref.hash[:8], + ) + } + + return nil + }, sqldb.NoOpReset, + ) + require.NoError(t, err) +} diff --git a/payments/db/migration1/testdata/README.md b/payments/db/migration1/testdata/README.md new file mode 100644 index 000000000..08c5fb06b --- /dev/null +++ b/payments/db/migration1/testdata/README.md @@ -0,0 +1,52 @@ +# Payment Migration External Testdata + +This directory holds a real `channel.db` (bbolt) or `channel.sqlite` file for +testing the payments KV to SQL migration locally. You can also point the test +at an existing Postgres-backed kvdb instance. + +## How to use + +1. Copy your `channel.db` or `channel.sqlite` file into this folder. +2. Edit `migration_external_test.go`: + + ```go + // Comment out this line to enable the test + t.Skipf("skipping test meant for local debugging only") + + // Set to your database filename + const fileName = "channel.db" // or "channel.sqlite" + ``` + +3. Run the test: + + ```bash + # For Postgres backend + go test -v -tags="test_db_postgres" -run TestMigrationWithExternalDB + ``` + +## SQLite kvdb source + +To migrate from a `channel.sqlite` file, run with the `kvdb_sqlite` build +tag: + +```bash +go test -v -tags="test_db_sqlite kvdb_sqlite" \ + -run TestMigrationWithExternalDB +``` + +## Postgres kvdb source + +To migrate from an existing Postgres-backed kvdb instance, edit +`postgresKVDSN` in `migration_external_test.go` (set it non-empty), then +run with the `kvdb_postgres` build tag: + +```bash +go test -v -tags="kvdb_postgres test_db_postgres" \ + -run TestMigrationWithExternalDB +``` + +## Notes + +- The external database is opened read-only. +- The test creates a fresh SQL database for each run. +- Do not commit production data; keep the file local. From f174b60b9424cceecf8405b9fc3a814ae6b33ad7 Mon Sep 17 00:00:00 2001 From: ziggie Date: Sun, 11 Jan 2026 00:09:42 +0100 Subject: [PATCH 07/16] =?UTF-8?q?payments/migration1:=20wire=20KV=E2=86=92?= =?UTF-8?q?SQL=20migration=20in=20the=20main=20pkg?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hook the payments KV→SQL migration into the SQL migration config. The migration is still only available when building with the build tag "test_native_sql". Moreover a tombstone protection similar to the invoice migration is added to prevent re-runningi with the KV backend once migration completes. --- config_builder.go | 58 ++++++++++++++++++++++++++++++ payments/db/kv_tombstone.go | 71 +++++++++++++++++++++++++++++++++++++ payments/db/log.go | 2 ++ sqldb/migrations_dev.go | 14 ++++++++ 4 files changed, 145 insertions(+) create mode 100644 payments/db/kv_tombstone.go diff --git a/config_builder.go b/config_builder.go index 74f8b3afb..729cb6017 100644 --- a/config_builder.go +++ b/config_builder.go @@ -51,6 +51,8 @@ import ( "github.com/lightningnetwork/lnd/macaroons" "github.com/lightningnetwork/lnd/msgmux" paymentsdb "github.com/lightningnetwork/lnd/payments/db" + paymentsmig1 "github.com/lightningnetwork/lnd/payments/db/migration1" + paymentsmig1sqlc "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" "github.com/lightningnetwork/lnd/rpcperms" "github.com/lightningnetwork/lnd/signal" "github.com/lightningnetwork/lnd/sqldb" @@ -76,6 +78,10 @@ const ( // graphMigration is the version number for the graph migration // that migrates the KV graph to the native SQL schema. graphMigration = 10 + + // paymentMigration is the version number for the payments migration + // that migrates KV payments to the native SQL schema. + paymentMigration = 14 ) // GrpcRegistrar is an interface that must be satisfied by an external subserver @@ -1153,6 +1159,31 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( return nil } + paymentMig := func(tx *sqlc.Queries) error { + err := paymentsmig1.MigratePaymentsKVToSQL( + ctx, + dbs.ChanStateDB.Backend, + paymentsmig1sqlc.New(tx.GetTx()), + &paymentsmig1.SQLStoreConfig{ + QueryCfg: queryCfg, + }, + ) + if err != nil { + return fmt.Errorf("failed to migrate "+ + "payments to SQL: %w", err) + } + + // Set the payments bucket tombstone to + // indicate that the migration has been + // completed. + d.logger.Debugf("Setting payments bucket " + + "tombstone") + + return paymentsdb.SetPaymentsBucketTombstone( + dbs.ChanStateDB.Backend, + ) + } + // Make sure we attach the custom migration function to // the correct migration version. for i := 0; i < len(migrations); i++ { @@ -1162,11 +1193,17 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( migrations[i].MigrationFn = invoiceMig continue + case graphMigration: migrations[i].MigrationFn = graphMig continue + case paymentMigration: + migrations[i].MigrationFn = paymentMig + + continue + default: } @@ -1265,6 +1302,27 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( return nil, nil, err } + // Check if the payments bucket tombstone is set. If it is, we + // need to return and ask the user switch back to using the + // native SQL store. + ripPayments, err := paymentsdb.GetPaymentsBucketTombstone( + dbs.ChanStateDB.Backend, + ) + if err != nil { + err = fmt.Errorf("unable to check payments bucket "+ + "tombstone: %w", err) + d.logger.Error(err) + + return nil, nil, err + } + if ripPayments { + err = fmt.Errorf("payments bucket tombstoned, please " + + "switch back to native SQL") + d.logger.Error(err) + + return nil, nil, err + } + dbs.InvoiceDB = dbs.ChanStateDB graphStore, err = graphdb.NewKVStore( diff --git a/payments/db/kv_tombstone.go b/payments/db/kv_tombstone.go new file mode 100644 index 000000000..7dc13b93b --- /dev/null +++ b/payments/db/kv_tombstone.go @@ -0,0 +1,71 @@ +package paymentsdb + +import ( + "fmt" + + "github.com/lightningnetwork/lnd/kvdb" +) + +var ( + // paymentsBucketTombstone is the key used to mark the payments bucket + // as permanently closed after a successful migration. + paymentsBucketTombstone = []byte("payments-tombstone") +) + +// SetPaymentsBucketTombstone sets the tombstone key in the payments bucket to +// mark the bucket as permanently closed. This prevents it from being reopened +// in the future. +func SetPaymentsBucketTombstone(db kvdb.Backend) error { + return kvdb.Update(db, func(tx kvdb.RwTx) error { + // Access the top-level payments bucket. + payments := tx.ReadWriteBucket(paymentsRootBucket) + + // In case the bucket doesn't exist, because we start + // immediately with the native SQL schema, we create it as well + // to make sure the user cannot switch back to the KV store. + if payments == nil { + var err error + payments, err = tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return fmt.Errorf("failed to create payments "+ + "bucket: %w", err) + } + } + + // Add the tombstone key to the payments bucket. + err := payments.Put(paymentsBucketTombstone, []byte("1")) + if err != nil { + return fmt.Errorf("failed to set tombstone: %w", err) + } + + return nil + }, func() {}) +} + +// GetPaymentsBucketTombstone checks if the tombstone key exists in the payments +// bucket. It returns true if the tombstone is present and false otherwise. +func GetPaymentsBucketTombstone(db kvdb.Backend) (bool, error) { + var tombstoneExists bool + + err := kvdb.View(db, func(tx kvdb.RTx) error { + // Access the top-level payments bucket. + payments := tx.ReadBucket(paymentsRootBucket) + if payments == nil { + tombstoneExists = false + return nil + } + + // Check if the tombstone key exists. + tombstone := payments.Get(paymentsBucketTombstone) + tombstoneExists = tombstone != nil + + return nil + }, func() {}) + if err != nil { + return false, err + } + + return tombstoneExists, nil +} diff --git a/payments/db/log.go b/payments/db/log.go index 8a77dbcec..c8892341d 100644 --- a/payments/db/log.go +++ b/payments/db/log.go @@ -3,6 +3,7 @@ package paymentsdb import ( "github.com/btcsuite/btclog/v2" "github.com/lightningnetwork/lnd/build" + paymentsmig1 "github.com/lightningnetwork/lnd/payments/db/migration1" ) // log is a logger that is initialized with no output filters. This @@ -29,4 +30,5 @@ func DisableLog() { // using btclog. func UseLogger(logger btclog.Logger) { log = logger + paymentsmig1.UseLogger(logger) } diff --git a/sqldb/migrations_dev.go b/sqldb/migrations_dev.go index ca57cf0f1..7b02b151f 100644 --- a/sqldb/migrations_dev.go +++ b/sqldb/migrations_dev.go @@ -8,4 +8,18 @@ var migrationAdditions = []MigrationConfig{ Version: 12, SchemaVersion: 10, }, + { + Name: "000011_payment_duplicates", + Version: 13, + SchemaVersion: 11, + }, + { + Name: "kv_payments_migration", + Version: 14, + SchemaVersion: 11, + // A migration function may be attached to this + // migration to migrate KV payments to the native SQL + // schema. This is optional and can be disabled by the + // user if necessary. + }, } From 89685de16ed2e2385ecf8c736b674cd62ccfac14 Mon Sep 17 00:00:00 2001 From: ziggie Date: Wed, 4 Feb 2026 14:42:42 -0500 Subject: [PATCH 08/16] payments/migration1: handle legacy payments with nil HTLC hash For legacy payments, the HTLC Hash field may be nil in the bbolt backend. Previously, the migration would fail with "HTLC attempt X missing payment hash" when encountering such payments. This commit fixes the migration by falling back to the parent payment hash when the HTLC-specific hash is nil. This is consistent with how the router handles legacy payments (see patchLegacyPaymentHash in payment_lifecycle.go). The validation logic is also updated to apply the same fallback when comparing bbolt data with migrated SQL data, ensuring the comparison succeeds. --- payments/db/migration1/migration_validation.go | 11 +++++++++++ payments/db/migration1/sql_migration.go | 17 ++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/payments/db/migration1/migration_validation.go b/payments/db/migration1/migration_validation.go index e97c3db40..e6f8bdd48 100644 --- a/payments/db/migration1/migration_validation.go +++ b/payments/db/migration1/migration_validation.go @@ -299,6 +299,17 @@ func normalizePaymentForCompare(payment *MPPayment) { htlc.circuit = nil htlc.cachedSessionKey = nil + // For legacy payments, the HTLC Hash field may be nil in the + // bbolt backend. During migration, the SQL code uses the + // parent payment hash as fallback. To ensure the comparison + // between bbolt and SQL data succeeds, we apply the same + // fallback here. + // + // See also: patchLegacyPaymentHash in payment_lifecycle.go. + if htlc.Hash == nil && payment.Info != nil { + htlc.Hash = &payment.Info.PaymentIdentifier + } + if len(htlc.Route.FirstHopWireCustomRecords) == 0 { htlc.Route.FirstHopWireCustomRecords = lnwire.CustomRecords{} diff --git a/payments/db/migration1/sql_migration.go b/payments/db/migration1/sql_migration.go index a7a02af5b..626b25a61 100644 --- a/payments/db/migration1/sql_migration.go +++ b/payments/db/migration1/sql_migration.go @@ -364,20 +364,23 @@ func migrateHTLCAttempt(ctx context.Context, paymentID int64, parentPaymentHash lntypes.Hash, htlc *HTLCAttempt, sqlDB SQLQueries, stats *MigrationStats) error { - // Validate that we have a payment hash for the attempt. + // Determine the payment hash for this HTLC attempt. // - // NOTE: We always require an attempt payment hash. A missing hash is an - // unrecoverable inconsistency. All payments should have a payment hash - // (AMP,MPP,Legacy) + // For AMP payments, each HTLC has its own unique hash. For non-AMP + // payments (MPP, Legacy), all HTLCs use the same hash as the parent + // payment. Older payment attempts may not have the hash stored + // explicitly, in which case we fall back to the parent payment hash + // which is ok since non-AMP payments have a single hash for all HTLCs. var paymentHash []byte switch { case htlc.Hash != nil: paymentHash = (*htlc.Hash)[:] default: - return fmt.Errorf("HTLC attempt %d missing payment hash "+ - "(parent payment hash=%x)", htlc.AttemptID, - parentPaymentHash[:]) + // For older payments where Hash is nil, use the parent payment + // hash. This is consistent with how the router handles these + // legacy payments. + paymentHash = parentPaymentHash[:] } firstHopAmountMsat := int64(htlc.Route.FirstHopAmount.Val.Int()) From c862e701481f17393132e5bf7109dc4fb4d25dfd Mon Sep 17 00:00:00 2001 From: ziggie Date: Sun, 11 Jan 2026 00:29:56 +0100 Subject: [PATCH 09/16] mod: update new direct dependency via go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 67c157080..7645bf62a 100644 --- a/go.mod +++ b/go.mod @@ -141,7 +141,7 @@ require ( github.com/opencontainers/runc v1.1.14 // indirect github.com/ory/dockertest/v3 v3.10.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect From 9def124380601422902f1d10483eb81271c49e06 Mon Sep 17 00:00:00 2001 From: ziggie Date: Mon, 16 Feb 2026 10:15:25 +0100 Subject: [PATCH 10/16] payments/db: fix silent error in duplicate payment lookup When duplicatePaymentSequenceKey is missing from a duplicate payment sub-bucket, the code returned the outer function's err variable which is nil at that point. This caused corrupted duplicate entries to be silently treated as "not found" instead of failing loudly. Return a new dedicated ErrNoDuplicateSequenceNumber error so malformed data is detected immediately. --- payments/db/errors.go | 5 +++++ payments/db/kv_store.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/payments/db/errors.go b/payments/db/errors.go index 0457db600..3b4e25bd9 100644 --- a/payments/db/errors.go +++ b/payments/db/errors.go @@ -136,6 +136,11 @@ var ( ErrNoDuplicateNestedBucket = errors.New("nested duplicate bucket not " + "found") + // ErrNoDuplicateSequenceNumber is returned when a duplicate payment + // sub-bucket does not contain the sequence number key. + ErrNoDuplicateSequenceNumber = errors.New("duplicate payment " + + "sequence number not found") + // ErrNoSequenceNrIndex is returned when an attempt to lookup a payment // index is made for a sequence number that is not indexed. // diff --git a/payments/db/kv_store.go b/payments/db/kv_store.go index 6d21048d7..b3fffe1e1 100644 --- a/payments/db/kv_store.go +++ b/payments/db/kv_store.go @@ -1236,7 +1236,7 @@ func fetchPaymentWithSequenceNumber(tx kvdb.RTx, paymentHash lntypes.Hash, seqBytes := subBucket.Get(duplicatePaymentSequenceKey) if seqBytes == nil { - return err + return ErrNoDuplicateSequenceNumber } // If this duplicate payment is not the sequence number we are From 932fbc33f0c2daed27352f36296c635f5fc42a85 Mon Sep 17 00:00:00 2001 From: ziggie Date: Mon, 16 Feb 2026 10:31:54 +0100 Subject: [PATCH 11/16] graph/db/migration1: fix defer commit/rollback in test tx executor The defer closure checked a local err variable for commit/rollback decisions, but err remained nil after a successful BeginTx. When txBody failed, the error was returned directly without assigning to err, so the defer always committed instead of rolling back. Additionally, since err was not a named return value, the defer's Commit error assignment was silently swallowed. Replace the error-prone defer pattern with explicit rollback on txBody failure and a direct Commit return. --- graph/db/migration1/test_sql.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/graph/db/migration1/test_sql.go b/graph/db/migration1/test_sql.go index 8034bec86..3d157dc9f 100644 --- a/graph/db/migration1/test_sql.go +++ b/graph/db/migration1/test_sql.go @@ -31,16 +31,15 @@ func (t *testBatchedSQLQueries) ExecTx(ctx context.Context, if err != nil { return err } - defer func() { - if err != nil { - _ = tx.Rollback() - } else { - err = tx.Commit() - } - }() reset() queries := sqlc.New(tx) - return txBody(queries) + if err := txBody(queries); err != nil { + _ = tx.Rollback() + + return err + } + + return tx.Commit() } From a2c36d166728ab2a320bc1f4fd88b7c8ac8dd428 Mon Sep 17 00:00:00 2001 From: ziggie Date: Mon, 16 Feb 2026 09:12:52 +0100 Subject: [PATCH 12/16] sqldb: add migration consistency test Tests that all migration files follow the defined schema and that there are not duplicates which could cause collision. --- sqldb/migrations_test.go | 97 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/sqldb/migrations_test.go b/sqldb/migrations_test.go index 68f111420..399afb1ce 100644 --- a/sqldb/migrations_test.go +++ b/sqldb/migrations_test.go @@ -616,3 +616,100 @@ func TestMigrationSucceedsAfterDirtyStateMigrationFailure19RC1(t *testing.T) { require.False(t, dirty) }) } + +// TestMigrationConfigConsistency verifies that the migration configuration in +// migrationConfig is consistent with the actual SQL schema files embedded in +// the binary. This catches version collisions (e.g. two migrations claiming +// the same schema version) and missing schema files. +func TestMigrationConfigConsistency(t *testing.T) { + t.Parallel() + + migrations := GetMigrations() + require.NotEmpty(t, migrations) + + // Build a set of schema versions that have actual .up.sql files in + // the embedded filesystem. + embeddedFiles, err := sqlSchemas.ReadDir("sqlc/migrations") + require.NoError(t, err) + + fileSchemaVersions := make(map[int]string) + for _, f := range embeddedFiles { + if f.IsDir() { + continue + } + + var version int + _, err := fmt.Sscanf(f.Name(), "%06d_", &version) + require.NoError(t, err, "schema migration file %q is "+ + "missing a valid numeric prefix (expected "+ + "format: 000XXX_name.up.sql)", f.Name()) + + // Enforce the 6-digit zero-padded naming convention + // for consistent directory listing order. + expectedPrefix := fmt.Sprintf("%06d_", version) + require.True(t, + len(f.Name()) > len(expectedPrefix) && + f.Name()[:len(expectedPrefix)] == expectedPrefix, + "schema migration file %q should use 6-digit "+ + "zero-padded prefix %q", f.Name(), + expectedPrefix) + + // Verify no two files share the same numeric prefix. + if existing, ok := fileSchemaVersions[version]; ok { + t.Fatalf("duplicate schema file version %06d: "+ + "%q and %q", version, existing, f.Name()) + } + + fileSchemaVersions[version] = f.Name() + } + + // Track seen versions to detect duplicates. + seenVersions := make(map[int]string) + seenSchemaVersions := make(map[int]string) + + for _, m := range migrations { + // 1. Verify no duplicate global versions. + if existing, ok := seenVersions[m.Version]; ok { + t.Fatalf("duplicate global version %d: %q and %q", + m.Version, existing, m.Name) + } + seenVersions[m.Version] = m.Name + + // 2. For schema migrations (those that advance the schema + // version), verify a corresponding .up.sql file exists + // and no two config entries claim the same schema version + // with different file prefixes. + prevSchema := 0 + if m.Version > 1 { + prevSchema = migrations[m.Version-2].SchemaVersion + } + + // A migration advances the schema if its SchemaVersion is + // higher than the previous migration's SchemaVersion. + if m.SchemaVersion > prevSchema { + _, hasFile := fileSchemaVersions[m.SchemaVersion] + require.True(t, hasFile, + "migration %q (version %d) declares "+ + "SchemaVersion=%d but no %06d_*.up.sql"+ + " file exists in the embedded FS", + m.Name, m.Version, m.SchemaVersion, + m.SchemaVersion) + + if existing, ok := seenSchemaVersions[m.SchemaVersion]; ok { + t.Fatalf("duplicate schema version %d: "+ + "%q and %q", m.SchemaVersion, + existing, m.Name) + } + seenSchemaVersions[m.SchemaVersion] = m.Name + } + } + + // 3. Verify versions are sequential starting from 1. + for i, m := range migrations { + require.Equal(t, i+1, m.Version, + "migration %q has version %d but expected %d "+ + "(migrations must be sequential)", + m.Name, m.Version, i+1) + } + +} From cf3610abb95506da66fd67b49f355685cb65021a Mon Sep 17 00:00:00 2001 From: ziggie Date: Tue, 24 Feb 2026 12:25:27 +0100 Subject: [PATCH 13/16] payments/db: split migration queries into dedicated SQLMigrationQueries interface --- payments/db/migration1/migration_external_test.go | 8 +++++++- payments/db/migration1/migration_validation.go | 2 +- payments/db/migration1/sql_migration.go | 8 ++++---- payments/db/migration1/sql_store.go | 15 +++++++++------ payments/db/sql_store.go | 15 +++++++++------ 5 files changed, 30 insertions(+), 18 deletions(-) diff --git a/payments/db/migration1/migration_external_test.go b/payments/db/migration1/migration_external_test.go index a5ddc42c8..85f828961 100644 --- a/payments/db/migration1/migration_external_test.go +++ b/payments/db/migration1/migration_external_test.go @@ -67,8 +67,14 @@ func TestMigrationWithExternalDB(t *testing.T) { // Run migration in a transaction. err := sqlStore.db.ExecTx( ctx, sqldb.WriteTxOpt(), func(tx SQLQueries) error { + migTx, ok := tx.(SQLMigrationQueries) + if !ok { + return fmt.Errorf("db does not " + + "implement SQLMigrationQueries") + } + return MigratePaymentsKVToSQL( - ctx, kvBackend, tx, &SQLStoreConfig{ + ctx, kvBackend, migTx, &SQLStoreConfig{ QueryCfg: sqlStore.cfg.QueryCfg, }, ) diff --git a/payments/db/migration1/migration_validation.go b/payments/db/migration1/migration_validation.go index e6f8bdd48..b2192e944 100644 --- a/payments/db/migration1/migration_validation.go +++ b/payments/db/migration1/migration_validation.go @@ -29,7 +29,7 @@ type migratedPaymentRef struct { // with their SQL counterparts. If a structural mismatch is detected, a full // deep comparison is performed to produce a detailed diff for debugging. func validateMigratedPaymentBatch(ctx context.Context, - kvBackend kvdb.Backend, sqlDB SQLQueries, + kvBackend kvdb.Backend, sqlDB SQLMigrationQueries, cfg *SQLStoreConfig, batch []migratedPaymentRef) error { if len(batch) == 0 { diff --git a/payments/db/migration1/sql_migration.go b/payments/db/migration1/sql_migration.go index 626b25a61..b20811760 100644 --- a/payments/db/migration1/sql_migration.go +++ b/payments/db/migration1/sql_migration.go @@ -37,7 +37,7 @@ type MigrationStats struct { // migrated data in batches. Callers are responsible for executing this within // a single SQL transaction if atomicity is required. func MigratePaymentsKVToSQL(ctx context.Context, kvBackend kvdb.Backend, - sqlDB SQLQueries, cfg *SQLStoreConfig) error { + sqlDB SQLMigrationQueries, cfg *SQLStoreConfig) error { if cfg == nil { return fmt.Errorf("missing SQL store config for migration") @@ -272,7 +272,7 @@ func normalizeTimeForSQL(t time.Time) time.Time { // migratePayment migrates a single payment from KV to SQL. func migratePayment(ctx context.Context, payment *MPPayment, hash lntypes.Hash, - sqlDB SQLQueries, stats *MigrationStats) (int64, error) { + sqlDB SQLMigrationQueries, stats *MigrationStats) (int64, error) { // Update migration stats based on payment status. switch payment.Status { @@ -602,7 +602,7 @@ func migrateRouteHop(ctx context.Context, // migrateDuplicatePayments migrates duplicate payments into the dedicated // payment_duplicates table. func migrateDuplicatePayments(ctx context.Context, dupBucket kvdb.RBucket, - hash [32]byte, primaryPaymentID int64, sqlDB SQLQueries, + hash [32]byte, primaryPaymentID int64, sqlDB SQLMigrationQueries, stats *MigrationStats) error { duplicateCount := 0 @@ -660,7 +660,7 @@ func migrateDuplicatePayments(ctx context.Context, dupBucket kvdb.RBucket, // given payment hash into payment_duplicates. func migrateSingleDuplicatePayment(ctx context.Context, dupBucket kvdb.RBucket, hash [32]byte, primaryPaymentID int64, duplicateSeq uint64, - sqlDB SQLQueries) error { + sqlDB SQLMigrationQueries) error { creationData := dupBucket.Get(duplicatePaymentCreationInfoKey) if creationData == nil { diff --git a/payments/db/migration1/sql_store.go b/payments/db/migration1/sql_store.go index 0fe4aa1fc..2a2c97752 100644 --- a/payments/db/migration1/sql_store.go +++ b/payments/db/migration1/sql_store.go @@ -88,13 +88,16 @@ type SQLQueries interface { // DeleteFailedAttempts removes all failed HTLCs from the db for a // given payment. DeleteFailedAttempts(ctx context.Context, paymentID int64) error +} - /* - Migration specific queries. - - These queries are used ONLY for the one-time migration from KV - to SQL. - */ +// SQLMigrationQueries extends SQLQueries with the additional queries needed +// for the one-time migration from KV to SQL. Keeping them in a separate +// interface makes it clear which code paths are migration-only and prevents +// the regular store from accidentally depending on them. +// +//nolint:ll +type SQLMigrationQueries interface { + SQLQueries // FetchPaymentsByIDsMig is a migration-only batch fetch that returns // payment data along with HTLC attempt counts for structural diff --git a/payments/db/sql_store.go b/payments/db/sql_store.go index 2c8dde93d..1c8ffb581 100644 --- a/payments/db/sql_store.go +++ b/payments/db/sql_store.go @@ -89,13 +89,16 @@ type SQLQueries interface { // DeleteFailedAttempts removes all failed HTLCs from the db for a // given payment. DeleteFailedAttempts(ctx context.Context, paymentID int64) error +} - /* - Migration specific queries. - - These queries are used ONLY for the one-time migration from KV - to SQL. - */ +// SQLMigrationQueries extends SQLQueries with the additional queries needed +// for the one-time migration from KV to SQL. Keeping them in a separate +// interface makes it clear which code paths are migration-only and prevents +// the regular store from accidentally depending on them. +// +//nolint:ll +type SQLMigrationQueries interface { + SQLQueries // FetchPaymentsByIDsMig is a migration-only batch fetch that returns // payment data along with HTLC attempt counts for structural From 00025ef3bf3d3b0311972815d8d84b4e8546fc7b Mon Sep 17 00:00:00 2001 From: ziggie Date: Thu, 26 Feb 2026 17:37:01 +0100 Subject: [PATCH 14/16] payments/db: normalize LegacyPayload flag in migration comparison MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LegacyPayload was a hint used exclusively by the KV store to decide how to serialize and deserialize the hop payload (legacy format vs TLV). The SQL store does not serialize hop data at all — every hop field is persisted natively in its own column — so this flag has no meaning there and is never stored. Clear LegacyPayload for all hops inside normalizePaymentForCompare so that deep-equality checks between KV and SQL payments succeed even when the KV source data carries LegacyPayload=true. A dedicated test (TestMigrationLegacyPayloadNormalized) is added to verify that a payment with LegacyPayload=true hops migrates and compares correctly. --- .../db/migration1/migration_validation.go | 9 + payments/db/migration1/sql_migration_test.go | 172 ++++++++++++++++++ 2 files changed, 181 insertions(+) diff --git a/payments/db/migration1/migration_validation.go b/payments/db/migration1/migration_validation.go index b2192e944..fe43cf28f 100644 --- a/payments/db/migration1/migration_validation.go +++ b/payments/db/migration1/migration_validation.go @@ -320,6 +320,15 @@ func normalizePaymentForCompare(payment *MPPayment) { htlc.Route.Hops[j].CustomRecords = record.CustomSet{} } + + // LegacyPayload was a hint used by the KV store to + // determine how to serialize and deserialize the hop + // payload (i.e. whether to use the legacy format or + // TLV). The SQL store does not serialize hop data at + // all — each field is stored natively in its own + // column — so this flag has no meaning there and is + // never persisted. + htlc.Route.Hops[j].LegacyPayload = false } } } diff --git a/payments/db/migration1/sql_migration_test.go b/payments/db/migration1/sql_migration_test.go index f1e96f025..f19dd9179 100644 --- a/payments/db/migration1/sql_migration_test.go +++ b/payments/db/migration1/sql_migration_test.go @@ -156,6 +156,178 @@ func TestMigrationDataIntegrity(t *testing.T) { } } +// TestMigrationLegacyPayloadNormalized verifies that a payment whose HTLC +// hops carry LegacyPayload=true in the KV store is migrated and compared +// correctly. LegacyPayload was a hint used by the KV store to choose between +// the legacy and TLV hop-payload serialization formats. The SQL store does not +// serialize hop data at all — each field is stored natively in its own +// column — so the flag is never persisted there. normalizePaymentForCompare +// clears it on both sides before the equality check, so the comparison must +// succeed even when the KV source has LegacyPayload=true. +func TestMigrationLegacyPayloadNormalized(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + kvDB := setupTestKVDB(t) + + hash := createTestPaymentHash(t, 0) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket( + paymentsIndexBucket, + ) + if err != nil { + return err + } + + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists( + hash[:], + ) + if err != nil { + return err + } + + var paymentID lntypes.Hash + copy(paymentID[:], hash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(1000000), + CreationTime: time.Now().Add(-24 * time.Hour), + PaymentRequest: []byte("lnbc1test"), + } + + var b bytes.Buffer + if err = serializePaymentCreationInfo( + &b, creationInfo, + ); err != nil { + return err + } + if err = paymentBucket.Put( + paymentCreationInfoKey, b.Bytes(), + ); err != nil { + return err + } + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, 0) + if err = paymentBucket.Put( + paymentSequenceKey, seqBytes, + ); err != nil { + return err + } + + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + hop1Key, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + hop2Key, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + sessionKey, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + const attemptID = uint64(1) + attemptInfo := &HTLCAttemptInfo{ + AttemptID: attemptID, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: 500000, + TotalAmount: lnwire.MilliSatoshi(1000000), + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex( + hop1Key.PubKey(), + ), + ChannelID: 123456, + OutgoingTimeLock: 499500, + AmtToForward: 900000, + LegacyPayload: true, + }, + { + PubKeyBytes: NewVertex( + hop2Key.PubKey(), + ), + ChannelID: 789012, + OutgoingTimeLock: 499000, + AmtToForward: 800000, + LegacyPayload: true, + }, + }, + }, + AttemptTime: time.Now().Add(-2 * time.Hour), + Hash: &paymentID, + } + + if err = writeHTLCAttempt( + htlcBucket, attemptID, attemptInfo, + ); err != nil { + return err + } + + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(paymentID), + SettleTime: time.Now().Add(-1 * time.Hour), + } + if err = writeHTLCSettle( + htlcBucket, attemptID, settleInfo, + ); err != nil { + return err + } + + return createIndexEntry(indexBucket, seqBytes, hash) + }, func() {}) + require.NoError(t, err) + + // Fetch the payment from KV and verify LegacyPayload is true so we + // know the test data is correct before migration. + var kvPayment *MPPayment + err = kvdb.View(kvDB, func(tx kvdb.RTx) error { + bucket := tx.ReadBucket(paymentsRootBucket). + NestedReadBucket(hash[:]) + var err error + kvPayment, err = fetchPayment(bucket) + return err + }, func() {}) + require.NoError(t, err) + require.True(t, kvPayment.HTLCs[0].Route.Hops[0].LegacyPayload) + require.True(t, kvPayment.HTLCs[0].Route.Hops[1].LegacyPayload) + + sqlStore := setupTestSQLDB(t) + require.NoError(t, runPaymentsMigration(ctx, kvDB, sqlStore)) + + // comparePaymentData normalizes both sides (clearing LegacyPayload) + // before the equality check, so this must pass. + comparePaymentData(t, ctx, sqlStore, kvPayment) +} + // TestMigrationWithDuplicates tests migration of duplicate payments into // the payment_duplicates table. func TestMigrationWithDuplicates(t *testing.T) { From 6c28cc4d0b4f368a0d79a2e410ae99e0a8ba2ecb Mon Sep 17 00:00:00 2001 From: ziggie Date: Fri, 27 Feb 2026 09:05:55 +0100 Subject: [PATCH 15/16] payments/migration1: freeze lnwire and record dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also freeze the lnwire and record packages used by the migration. Copy the minimal subset of lnwire files (16) into payments/db/migration1/lnwire/ and all record files (6) into payments/db/migration1/record/. Three lnwire files are trimmed to avoid pulling in the full message-type dispatch tree — all changes are purely subtractive and can be verified with: diff lnwire/message.go payments/db/migration1/lnwire/message.go diff lnwire/writer.go payments/db/migration1/lnwire/writer.go diff lnwire/lnwire.go payments/db/migration1/lnwire/lnwire.go All migration1 files now import only the frozen packages, removing the live dependency on lnwire and record so future changes to those packages cannot affect migration correctness. --- .golangci.yml | 4 +- payments/db/migration1/codec.go | 2 +- .../db/migration1/kv_duplicate_payments.go | 2 +- payments/db/migration1/kv_store.go | 4 +- payments/db/migration1/lnwire/channel_id.go | 127 ++ .../db/migration1/lnwire/channel_update.go | 422 +++++ .../db/migration1/lnwire/custom_records.go | 278 +++ payments/db/migration1/lnwire/error.go | 143 ++ payments/db/migration1/lnwire/extra_bytes.go | 309 ++++ payments/db/migration1/lnwire/features.go | 898 ++++++++++ payments/db/migration1/lnwire/interfaces.go | 174 ++ payments/db/migration1/lnwire/lnwire.go | 426 +++++ payments/db/migration1/lnwire/message.go | 323 ++++ payments/db/migration1/lnwire/msat.go | 92 + payments/db/migration1/lnwire/onion_error.go | 1546 +++++++++++++++++ .../db/migration1/lnwire/short_channel_id.go | 105 ++ payments/db/migration1/lnwire/signature.go | 292 ++++ payments/db/migration1/lnwire/typed_fee.go | 60 + .../db/migration1/lnwire/update_add_htlc.go | 226 +++ payments/db/migration1/lnwire/writer.go | 185 ++ .../db/migration1/migration_validation.go | 4 +- payments/db/migration1/payment.go | 2 +- payments/db/migration1/record/amp.go | 121 ++ payments/db/migration1/record/blinded_data.go | 440 +++++ .../db/migration1/record/custom_records.go | 31 + payments/db/migration1/record/experimental.go | 6 + payments/db/migration1/record/hop.go | 99 ++ payments/db/migration1/record/mpp.go | 112 ++ payments/db/migration1/route.go | 4 +- payments/db/migration1/sql_converters.go | 4 +- payments/db/migration1/sql_migration.go | 2 +- payments/db/migration1/sql_migration_test.go | 4 +- payments/db/migration1/sql_store.go | 2 +- 33 files changed, 6433 insertions(+), 16 deletions(-) create mode 100644 payments/db/migration1/lnwire/channel_id.go create mode 100644 payments/db/migration1/lnwire/channel_update.go create mode 100644 payments/db/migration1/lnwire/custom_records.go create mode 100644 payments/db/migration1/lnwire/error.go create mode 100644 payments/db/migration1/lnwire/extra_bytes.go create mode 100644 payments/db/migration1/lnwire/features.go create mode 100644 payments/db/migration1/lnwire/interfaces.go create mode 100644 payments/db/migration1/lnwire/lnwire.go create mode 100644 payments/db/migration1/lnwire/message.go create mode 100644 payments/db/migration1/lnwire/msat.go create mode 100644 payments/db/migration1/lnwire/onion_error.go create mode 100644 payments/db/migration1/lnwire/short_channel_id.go create mode 100644 payments/db/migration1/lnwire/signature.go create mode 100644 payments/db/migration1/lnwire/typed_fee.go create mode 100644 payments/db/migration1/lnwire/update_add_htlc.go create mode 100644 payments/db/migration1/lnwire/writer.go create mode 100644 payments/db/migration1/record/amp.go create mode 100644 payments/db/migration1/record/blinded_data.go create mode 100644 payments/db/migration1/record/custom_records.go create mode 100644 payments/db/migration1/record/experimental.go create mode 100644 payments/db/migration1/record/hop.go create mode 100644 payments/db/migration1/record/mpp.go diff --git a/.golangci.yml b/.golangci.yml index dc82da888..4cb893934 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -302,7 +302,9 @@ linters: - "internal\\/musig2v040" - channeldb/migration_01_to_11 - channeldb/migration/lnwire21 - + - payments/db/migration1/lnwire + - payments/db/migration1/record + issues: # Only show newly introduced problems. new-from-rev: 03eab4db64540aa5f789c617793e4459f4ba9e78 diff --git a/payments/db/migration1/codec.go b/payments/db/migration1/codec.go index acc173bc4..64225d170 100644 --- a/payments/db/migration1/codec.go +++ b/payments/db/migration1/codec.go @@ -7,7 +7,7 @@ import ( "time" "github.com/btcsuite/btcd/wire" - "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" ) // Big endian is the preferred byte order, due to cursor scans over diff --git a/payments/db/migration1/kv_duplicate_payments.go b/payments/db/migration1/kv_duplicate_payments.go index f72848da4..1454430bc 100644 --- a/payments/db/migration1/kv_duplicate_payments.go +++ b/payments/db/migration1/kv_duplicate_payments.go @@ -10,7 +10,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/lightningnetwork/lnd/kvdb" "github.com/lightningnetwork/lnd/lntypes" - "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" ) var ( diff --git a/payments/db/migration1/kv_store.go b/payments/db/migration1/kv_store.go index 51bf005db..eda1945bc 100644 --- a/payments/db/migration1/kv_store.go +++ b/payments/db/migration1/kv_store.go @@ -17,8 +17,8 @@ import ( "github.com/lightningnetwork/lnd/channeldb" "github.com/lightningnetwork/lnd/kvdb" "github.com/lightningnetwork/lnd/lntypes" - "github.com/lightningnetwork/lnd/lnwire" - "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/record" "github.com/lightningnetwork/lnd/tlv" ) diff --git a/payments/db/migration1/lnwire/channel_id.go b/payments/db/migration1/lnwire/channel_id.go new file mode 100644 index 000000000..5c9eca34f --- /dev/null +++ b/payments/db/migration1/lnwire/channel_id.go @@ -0,0 +1,127 @@ +package lnwire + +import ( + "encoding/binary" + "encoding/hex" + "io" + "math" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // MaxFundingTxOutputs is the maximum number of allowed outputs on a + // funding transaction within the protocol. This is due to the fact + // that we use 2-bytes to encode the index within the funding output + // during the funding workflow. Funding transaction with more outputs + // than this are considered invalid within the protocol. + MaxFundingTxOutputs = math.MaxUint16 +) + +// ChannelID is a series of 32-bytes that uniquely identifies all channels +// within the network. The ChannelID is computed using the outpoint of the +// funding transaction (the txid, and output index). Given a funding output the +// ChannelID can be calculated by XOR'ing the big-endian serialization of the +// txid and the big-endian serialization of the output index, truncated to +// 2 bytes. +type ChannelID [32]byte + +// ConnectionWideID is an all-zero ChannelID, which is used to represent a +// message intended for all channels to specific peer. +var ConnectionWideID = ChannelID{} + +// String returns the string representation of the ChannelID. This is just the +// hex string encoding of the ChannelID itself. +func (c ChannelID) String() string { + return hex.EncodeToString(c[:]) +} + +// Record returns a TLV record that can be used to encode/decode a ChannelID +// to/from a TLV stream. +func (c *ChannelID) Record() tlv.Record { + return tlv.MakeStaticRecord(0, c, 32, encodeChannelID, decodeChannelID) +} + +func encodeChannelID(w io.Writer, val interface{}, buf *[8]byte) error { + if v, ok := val.(*ChannelID); ok { + bigSize := [32]byte(*v) + + return tlv.EBytes32(w, &bigSize, buf) + } + + return tlv.NewTypeForEncodingErr(val, "lnwire.ChannelID") +} + +func decodeChannelID(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if v, ok := val.(*ChannelID); ok { + var id [32]byte + err := tlv.DBytes32(r, &id, buf, l) + if err != nil { + return err + } + + *v = id + + return nil + } + + return tlv.NewTypeForDecodingErr(val, "lnwire.ChannelID", l, l) +} + +// NewChanIDFromOutPoint converts a target OutPoint into a ChannelID that is +// usable within the network. In order to convert the OutPoint into a ChannelID, +// we XOR the lower 2-bytes of the txid within the OutPoint with the big-endian +// serialization of the Index of the OutPoint, truncated to 2-bytes. +func NewChanIDFromOutPoint(op wire.OutPoint) ChannelID { + // First we'll copy the txid of the outpoint into our channel ID slice. + var cid ChannelID + copy(cid[:], op.Hash[:]) + + // With the txid copied over, we'll now XOR the lower 2-bytes of the + // partial channelID with big-endian serialization of output index. + xorTxid(&cid, uint16(op.Index)) + + return cid +} + +// xorTxid performs the transformation needed to transform an OutPoint into a +// ChannelID. To do this, we expect the cid parameter to contain the txid +// unaltered and the outputIndex to be the output index +func xorTxid(cid *ChannelID, outputIndex uint16) { + var buf [2]byte + binary.BigEndian.PutUint16(buf[:], outputIndex) + + cid[30] ^= buf[0] + cid[31] ^= buf[1] +} + +// GenPossibleOutPoints generates all the possible outputs given a channel ID. +// In order to generate these possible outpoints, we perform a brute-force +// search through the candidate output index space, performing a reverse +// mapping from channelID back to OutPoint. +func (c *ChannelID) GenPossibleOutPoints() [MaxFundingTxOutputs]wire.OutPoint { + var possiblePoints [MaxFundingTxOutputs]wire.OutPoint + for i := uint16(0); i < MaxFundingTxOutputs; i++ { + cidCopy := *c + xorTxid(&cidCopy, i) + + possiblePoints[i] = wire.OutPoint{ + Hash: chainhash.Hash(cidCopy), + Index: uint32(i), + } + } + + return possiblePoints +} + +// IsChanPoint returns true if the OutPoint passed corresponds to the target +// ChannelID. +func (c ChannelID) IsChanPoint(op *wire.OutPoint) bool { + candidateCid := NewChanIDFromOutPoint(*op) + + return candidateCid == c +} diff --git a/payments/db/migration1/lnwire/channel_update.go b/payments/db/migration1/lnwire/channel_update.go new file mode 100644 index 000000000..09b8044bd --- /dev/null +++ b/payments/db/migration1/lnwire/channel_update.go @@ -0,0 +1,422 @@ +package lnwire + +import ( + "bytes" + "fmt" + "io" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/lightningnetwork/lnd/tlv" +) + +// ChanUpdateMsgFlags is a bitfield that signals whether optional fields are +// present in the ChannelUpdate. +type ChanUpdateMsgFlags uint8 + +const ( + // ChanUpdateRequiredMaxHtlc is a bit that indicates whether the + // required htlc_maximum_msat field is present in this ChannelUpdate. + ChanUpdateRequiredMaxHtlc ChanUpdateMsgFlags = 1 << iota +) + +// String returns the bitfield flags as a string. +func (c ChanUpdateMsgFlags) String() string { + return fmt.Sprintf("%08b", c) +} + +// HasMaxHtlc returns true if the htlc_maximum_msat option bit is set in the +// message flags. +func (c ChanUpdateMsgFlags) HasMaxHtlc() bool { + return c&ChanUpdateRequiredMaxHtlc != 0 +} + +// ChanUpdateChanFlags is a bitfield that signals various options concerning a +// particular channel edge. Each bit is to be examined in order to determine +// how the ChannelUpdate message is to be interpreted. +type ChanUpdateChanFlags uint8 + +const ( + // ChanUpdateDirection indicates the direction of a channel update. If + // this bit is set to 0 if Node1 (the node with the "smaller" Node ID) + // is updating the channel, and to 1 otherwise. + ChanUpdateDirection ChanUpdateChanFlags = 1 << iota + + // ChanUpdateDisabled is a bit that indicates if the channel edge + // selected by the ChanUpdateDirection bit is to be treated as being + // disabled. + ChanUpdateDisabled +) + +// IsDisabled determines whether the channel flags has the disabled bit set. +func (c ChanUpdateChanFlags) IsDisabled() bool { + return c&ChanUpdateDisabled == ChanUpdateDisabled +} + +// String returns the bitfield flags as a string. +func (c ChanUpdateChanFlags) String() string { + return fmt.Sprintf("%08b", c) +} + +// ChannelUpdate1 message is used after channel has been initially announced. +// Each side independently announces its fees and minimum expiry for HTLCs and +// other parameters. Also this message is used to redeclare initially set +// channel parameters. +type ChannelUpdate1 struct { + // Signature is used to validate the announced data and prove the + // ownership of node id. + Signature Sig + + // ChainHash denotes the target chain that this channel was opened + // within. This value should be the genesis hash of the target chain. + // Along with the short channel ID, this uniquely identifies the + // channel globally in a blockchain. + ChainHash chainhash.Hash + + // ShortChannelID is the unique description of the funding transaction. + ShortChannelID ShortChannelID + + // Timestamp allows ordering in the case of multiple announcements. We + // should ignore the message if timestamp is not greater than + // the last-received. + Timestamp uint32 + + // MessageFlags is a bitfield that describes whether optional fields + // are present in this update. Currently, the least-significant bit + // must be set to 1 if the optional field MaxHtlc is present. + MessageFlags ChanUpdateMsgFlags + + // ChannelFlags is a bitfield that describes additional meta-data + // concerning how the update is to be interpreted. Currently, the + // least-significant bit must be set to 0 if the creating node + // corresponds to the first node in the previously sent channel + // announcement and 1 otherwise. If the second bit is set, then the + // channel is set to be disabled. + ChannelFlags ChanUpdateChanFlags + + // TimeLockDelta is the minimum number of blocks this node requires to + // be added to the expiry of HTLCs. This is a security parameter + // determined by the node operator. This value represents the required + // gap between the time locks of the incoming and outgoing HTLC's set + // to this node. + TimeLockDelta uint16 + + // HtlcMinimumMsat is the minimum HTLC value which will be accepted. + HtlcMinimumMsat MilliSatoshi + + // BaseFee is the base fee that must be used for incoming HTLC's to + // this particular channel. This value will be tacked onto the required + // for a payment independent of the size of the payment. + BaseFee uint32 + + // FeeRate is the fee rate that will be charged per millionth of a + // satoshi. + FeeRate uint32 + + // HtlcMaximumMsat is the maximum HTLC value which will be accepted. + HtlcMaximumMsat MilliSatoshi + + // InboundFee is an optional TLV record that contains the fee + // information for incoming HTLCs. + InboundFee tlv.OptionalRecordT[tlv.TlvType55555, Fee] + + // ExtraData is the set of data that was appended to this message to + // fill out the full maximum transport message size. These fields can + // be used to specify optional data such as custom TLV fields. + ExtraOpaqueData ExtraOpaqueData +} + +// A compile time check to ensure ChannelUpdate implements the lnwire.Message +// interface. +var _ Message = (*ChannelUpdate1)(nil) + +// A compile time check to ensure ChannelUpdate1 implements the +// lnwire.SizeableMessage interface. +var _ SizeableMessage = (*ChannelUpdate1)(nil) + +// Decode deserializes a serialized ChannelUpdate stored in the passed +// io.Reader observing the specified protocol version. +// +// This is part of the lnwire.Message interface. +func (a *ChannelUpdate1) Decode(r io.Reader, _ uint32) error { + err := ReadElements(r, + &a.Signature, + a.ChainHash[:], + &a.ShortChannelID, + &a.Timestamp, + &a.MessageFlags, + &a.ChannelFlags, + &a.TimeLockDelta, + &a.HtlcMinimumMsat, + &a.BaseFee, + &a.FeeRate, + ) + if err != nil { + return err + } + + // Now check whether the max HTLC field is present and read it if so. + if a.MessageFlags.HasMaxHtlc() { + if err := ReadElements(r, &a.HtlcMaximumMsat); err != nil { + return err + } + } + + var tlvRecords ExtraOpaqueData + if err := ReadElements(r, &tlvRecords); err != nil { + return err + } + + var inboundFee = a.InboundFee.Zero() + typeMap, err := tlvRecords.ExtractRecords(&inboundFee) + if err != nil { + return fmt.Errorf("%w: %w", ErrParsingExtraTLVBytes, err) + } + + val, ok := typeMap[a.InboundFee.TlvType()] + if ok && val == nil { + a.InboundFee = tlv.SomeRecordT(inboundFee) + } + + if len(tlvRecords) != 0 { + a.ExtraOpaqueData = tlvRecords + } + + return nil +} + +// Encode serializes the target ChannelUpdate into the passed io.Writer +// observing the protocol version specified. +// +// This is part of the lnwire.Message interface. +func (a *ChannelUpdate1) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteSig(w, a.Signature); err != nil { + return err + } + + if err := WriteBytes(w, a.ChainHash[:]); err != nil { + return err + } + + if err := WriteShortChannelID(w, a.ShortChannelID); err != nil { + return err + } + + if err := WriteUint32(w, a.Timestamp); err != nil { + return err + } + + if err := WriteChanUpdateMsgFlags(w, a.MessageFlags); err != nil { + return err + } + + if err := WriteChanUpdateChanFlags(w, a.ChannelFlags); err != nil { + return err + } + + if err := WriteUint16(w, a.TimeLockDelta); err != nil { + return err + } + + if err := WriteMilliSatoshi(w, a.HtlcMinimumMsat); err != nil { + return err + } + + if err := WriteUint32(w, a.BaseFee); err != nil { + return err + } + + if err := WriteUint32(w, a.FeeRate); err != nil { + return err + } + + // Now append optional fields if they are set. Currently, the only + // optional field is max HTLC. + if a.MessageFlags.HasMaxHtlc() { + err := WriteMilliSatoshi(w, a.HtlcMaximumMsat) + if err != nil { + return err + } + } + + recordProducers := make([]tlv.RecordProducer, 0, 1) + a.InboundFee.WhenSome(func(fee tlv.RecordT[tlv.TlvType55555, Fee]) { + recordProducers = append(recordProducers, &fee) + }) + + err := EncodeMessageExtraData(&a.ExtraOpaqueData, recordProducers...) + if err != nil { + return err + } + + // Finally, append any extra opaque data. + return WriteBytes(w, a.ExtraOpaqueData) +} + +// MsgType returns the integer uniquely identifying this message type on the +// wire. +// +// This is part of the lnwire.Message interface. +func (a *ChannelUpdate1) MsgType() MessageType { + return MsgChannelUpdate +} + +// DataToSign is used to retrieve part of the announcement message which should +// be signed. +func (a *ChannelUpdate1) DataToSign() ([]byte, error) { + // We should not include the signatures itself. + b := make([]byte, 0, MaxMsgBody) + buf := bytes.NewBuffer(b) + if err := WriteBytes(buf, a.ChainHash[:]); err != nil { + return nil, err + } + + if err := WriteShortChannelID(buf, a.ShortChannelID); err != nil { + return nil, err + } + + if err := WriteUint32(buf, a.Timestamp); err != nil { + return nil, err + } + + if err := WriteChanUpdateMsgFlags(buf, a.MessageFlags); err != nil { + return nil, err + } + + if err := WriteChanUpdateChanFlags(buf, a.ChannelFlags); err != nil { + return nil, err + } + + if err := WriteUint16(buf, a.TimeLockDelta); err != nil { + return nil, err + } + + if err := WriteMilliSatoshi(buf, a.HtlcMinimumMsat); err != nil { + return nil, err + } + + if err := WriteUint32(buf, a.BaseFee); err != nil { + return nil, err + } + + if err := WriteUint32(buf, a.FeeRate); err != nil { + return nil, err + } + + // Now append optional fields if they are set. Currently, the only + // optional field is max HTLC. + if a.MessageFlags.HasMaxHtlc() { + err := WriteMilliSatoshi(buf, a.HtlcMaximumMsat) + if err != nil { + return nil, err + } + } + + // Finally, append any extra opaque data. + if err := WriteBytes(buf, a.ExtraOpaqueData); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// SCID returns the ShortChannelID of the channel that the update applies to. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) SCID() ShortChannelID { + return a.ShortChannelID +} + +// IsNode1 is true if the update was produced by node 1 of the channel peers. +// Node 1 is the node with the lexicographically smaller public key. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) IsNode1() bool { + return a.ChannelFlags&ChanUpdateDirection == 0 +} + +// IsDisabled is true if the update is announcing that the channel should be +// considered disabled. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) IsDisabled() bool { + return a.ChannelFlags&ChanUpdateDisabled == ChanUpdateDisabled +} + +// GetChainHash returns the hash of the chain that the message is referring to. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) GetChainHash() chainhash.Hash { + return a.ChainHash +} + +// ForwardingPolicy returns the set of forwarding constraints of the update. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) ForwardingPolicy() *ForwardingPolicy { + return &ForwardingPolicy{ + TimeLockDelta: a.TimeLockDelta, + BaseFee: MilliSatoshi(a.BaseFee), + FeeRate: MilliSatoshi(a.FeeRate), + MinHTLC: a.HtlcMinimumMsat, + HasMaxHTLC: a.MessageFlags.HasMaxHtlc(), + MaxHTLC: a.HtlcMaximumMsat, + } +} + +// GossipVersion returns the gossip version that this message is part of. +// +// NOTE: this is part of the GossipMessage interface. +func (a *ChannelUpdate1) GossipVersion() GossipVersion { + return GossipVersion1 +} + +// CmpAge can be used to determine if the update is older or newer than the +// passed update. It returns 1 if this update is newer, -1 if it is older, and +// 0 if they are the same age. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) CmpAge(update ChannelUpdate) (CompareResult, error) { + other, ok := update.(*ChannelUpdate1) + if !ok { + return 0, fmt.Errorf("expected *ChannelUpdate1, got: %T", + update) + } + + switch { + case a.Timestamp > other.Timestamp: + return GreaterThan, nil + case a.Timestamp < other.Timestamp: + return LessThan, nil + default: + return EqualTo, nil + } +} + +// SetDisabledFlag can be used to adjust the disabled flag of an update. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) SetDisabledFlag(disabled bool) { + if disabled { + a.ChannelFlags |= ChanUpdateDisabled + } else { + a.ChannelFlags &= ^ChanUpdateDisabled + } +} + +// SetSCID can be used to overwrite the SCID of the update. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) SetSCID(scid ShortChannelID) { + a.ShortChannelID = scid +} + +// A compile time assertion to ensure ChannelUpdate1 implements the +// ChannelUpdate interface. +var _ ChannelUpdate = (*ChannelUpdate1)(nil) + +// SerializedSize returns the serialized size of the message in bytes. +// +// This is part of the lnwire.SizeableMessage interface. +func (a *ChannelUpdate1) SerializedSize() (uint32, error) { + return MessageSerializedSize(a) +} diff --git a/payments/db/migration1/lnwire/custom_records.go b/payments/db/migration1/lnwire/custom_records.go new file mode 100644 index 000000000..de5ff4a23 --- /dev/null +++ b/payments/db/migration1/lnwire/custom_records.go @@ -0,0 +1,278 @@ +package lnwire + +import ( + "bytes" + "fmt" + "io" + "maps" + "sort" + + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // MinCustomRecordsTlvType is the minimum custom records TLV type as + // defined in BOLT 01. + MinCustomRecordsTlvType = 65536 +) + +// CustomRecords stores a set of custom key/value pairs. Map keys are TLV types +// which must be greater than or equal to MinCustomRecordsTlvType. +type CustomRecords map[uint64][]byte + +// NewCustomRecords creates a new CustomRecords instance from a +// tlv.TypeMap. +func NewCustomRecords(tlvMap tlv.TypeMap) (CustomRecords, error) { + // Make comparisons in unit tests easy by returning nil if the map is + // empty. + if len(tlvMap) == 0 { + return nil, nil + } + + customRecords := make(CustomRecords, len(tlvMap)) + for k, v := range tlvMap { + customRecords[uint64(k)] = v + } + + // Validate the custom records. + err := customRecords.Validate() + if err != nil { + return nil, fmt.Errorf("custom records from tlv map "+ + "validation error: %w", err) + } + + return customRecords, nil +} + +// ParseCustomRecords creates a new CustomRecords instance from a tlv.Blob. +func ParseCustomRecords(b tlv.Blob) (CustomRecords, error) { + return ParseCustomRecordsFrom(bytes.NewReader(b)) +} + +// ParseCustomRecordsFrom creates a new CustomRecords instance from a reader. +func ParseCustomRecordsFrom(r io.Reader) (CustomRecords, error) { + typeMap, err := DecodeRecords(r) + if err != nil { + return nil, fmt.Errorf("error decoding HTLC record: %w", err) + } + + return NewCustomRecords(typeMap) +} + +// Validate checks that all custom records are in the custom type range. +func (c CustomRecords) Validate() error { + if c == nil { + return nil + } + + for key := range c { + if key < MinCustomRecordsTlvType { + return fmt.Errorf("custom records entry with TLV "+ + "type below min: %d", MinCustomRecordsTlvType) + } + } + + return nil +} + +// Copy returns a copy of the custom records. +func (c CustomRecords) Copy() CustomRecords { + if c == nil { + return nil + } + + customRecords := make(CustomRecords, len(c)) + for k, v := range c { + customRecords[k] = v + } + + return customRecords +} + +// MergedCopy creates a copy of the records and merges them with the given +// records. If the same key is present in both sets, the value from the other +// records will be used. +func (c CustomRecords) MergedCopy(other CustomRecords) CustomRecords { + copiedRecords := make(CustomRecords, len(c)) + maps.Copy(copiedRecords, c) + maps.Copy(copiedRecords, other) + + return copiedRecords +} + +// ExtendRecordProducers extends the given records slice with the custom +// records. The resultant records slice will be sorted if the given records +// slice contains TLV types greater than or equal to MinCustomRecordsTlvType. +func (c CustomRecords) ExtendRecordProducers( + producers []tlv.RecordProducer) ([]tlv.RecordProducer, error) { + + // If the custom records are nil or empty, there is nothing to do. + if len(c) == 0 { + return producers, nil + } + + // Validate the custom records. + err := c.Validate() + if err != nil { + return nil, err + } + + // Ensure that the existing records slice TLV types are not also present + // in the custom records. If they are, the resultant extended records + // slice would erroneously contain duplicate TLV types. + for _, rp := range producers { + record := rp.Record() + recordTlvType := uint64(record.Type()) + + _, foundDuplicateTlvType := c[recordTlvType] + if foundDuplicateTlvType { + return nil, fmt.Errorf("custom records contains a TLV "+ + "type that is already present in the "+ + "existing records: %d", recordTlvType) + } + } + + // Convert the custom records map to a TLV record producer slice and + // append them to the exiting records slice. + customRecordProducers := RecordsAsProducers(tlv.MapToRecords(c)) + producers = append(producers, customRecordProducers...) + + // If the records slice which was given as an argument included TLV + // values greater than or equal to the minimum custom records TLV type + // we will sort the extended records slice to ensure that it is ordered + // correctly. + SortProducers(producers) + + return producers, nil +} + +// RecordProducers returns a slice of record producers for the custom records. +func (c CustomRecords) RecordProducers() []tlv.RecordProducer { + // If the custom records are nil or empty, return an empty slice. + if len(c) == 0 { + return nil + } + + // Convert the custom records map to a TLV record producer slice. + records := tlv.MapToRecords(c) + + return RecordsAsProducers(records) +} + +// Serialize serializes the custom records into a byte slice. +func (c CustomRecords) Serialize() ([]byte, error) { + records := tlv.MapToRecords(c) + return EncodeRecords(records) +} + +// SerializeTo serializes the custom records into the given writer. +func (c CustomRecords) SerializeTo(w io.Writer) error { + records := tlv.MapToRecords(c) + return EncodeRecordsTo(w, records) +} + +// ProduceRecordsSorted converts a slice of record producers into a slice of +// records and then sorts it by type. +func ProduceRecordsSorted(recordProducers ...tlv.RecordProducer) []tlv.Record { + records := fn.Map( + recordProducers, + func(producer tlv.RecordProducer) tlv.Record { + return producer.Record() + }, + ) + + // Ensure that the set of records are sorted before we attempt to + // decode from the stream, to ensure they're canonical. + tlv.SortRecords(records) + + return records +} + +// SortProducers sorts the given record producers by their type. +func SortProducers(producers []tlv.RecordProducer) { + sort.Slice(producers, func(i, j int) bool { + recordI := producers[i].Record() + recordJ := producers[j].Record() + return recordI.Type() < recordJ.Type() + }) +} + +// TlvMapToRecords converts a TLV map into a slice of records. +func TlvMapToRecords(tlvMap tlv.TypeMap) []tlv.Record { + tlvMapGeneric := make(map[uint64][]byte) + for k, v := range tlvMap { + tlvMapGeneric[uint64(k)] = v + } + + return tlv.MapToRecords(tlvMapGeneric) +} + +// RecordsAsProducers converts a slice of records into a slice of record +// producers. +func RecordsAsProducers(records []tlv.Record) []tlv.RecordProducer { + return fn.Map(records, func(record tlv.Record) tlv.RecordProducer { + return &record + }) +} + +// EncodeRecords encodes the given records into a byte slice. +func EncodeRecords(records []tlv.Record) ([]byte, error) { + var buf bytes.Buffer + if err := EncodeRecordsTo(&buf, records); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// EncodeRecordsTo encodes the given records into the given writer. +func EncodeRecordsTo(w io.Writer, records []tlv.Record) error { + tlvStream, err := tlv.NewStream(records...) + if err != nil { + return err + } + + return tlvStream.Encode(w) +} + +// DecodeRecords decodes the given byte slice into the given records and returns +// the rest as a TLV type map. +func DecodeRecords(r io.Reader, + records ...tlv.Record) (tlv.TypeMap, error) { + + tlvStream, err := tlv.NewStream(records...) + if err != nil { + return nil, err + } + + return tlvStream.DecodeWithParsedTypes(r) +} + +// DecodeRecordsP2P decodes the given byte slice into the given records and +// returns the rest as a TLV type map. This function is identical to +// DecodeRecords except that the record size is capped at 65535. +func DecodeRecordsP2P(r *bytes.Reader, + records ...tlv.Record) (tlv.TypeMap, error) { + + tlvStream, err := tlv.NewStream(records...) + if err != nil { + return nil, err + } + + return tlvStream.DecodeWithParsedTypesP2P(r) +} + +// AssertUniqueTypes asserts that the given records have unique types. +func AssertUniqueTypes(r []tlv.Record) error { + seen := make(fn.Set[tlv.Type], len(r)) + for _, record := range r { + t := record.Type() + if seen.Contains(t) { + return fmt.Errorf("duplicate record type: %d", t) + } + seen.Add(t) + } + + return nil +} diff --git a/payments/db/migration1/lnwire/error.go b/payments/db/migration1/lnwire/error.go new file mode 100644 index 000000000..382411999 --- /dev/null +++ b/payments/db/migration1/lnwire/error.go @@ -0,0 +1,143 @@ +package lnwire + +import ( + "bytes" + "fmt" + "io" +) + +var ( + // ErrParsingExtraTLVBytes is returned when we attempt to parse + // extra opaque bytes as a TLV stream, but the parsing fails due to + // and invalid TLV stream. + ErrParsingExtraTLVBytes = fmt.Errorf("error parsing extra TLV bytes") +) + +// FundingError represents a set of errors that can be encountered and sent +// during the funding workflow. +type FundingError uint8 + +const ( + // ErrMaxPendingChannels is returned by remote peer when the number of + // active pending channels exceeds their maximum policy limit. + ErrMaxPendingChannels FundingError = 1 + + // ErrChanTooLarge is returned by a remote peer that receives a + // FundingOpen request for a channel that is above their current + // soft-limit. + ErrChanTooLarge FundingError = 2 +) + +// String returns a human readable version of the target FundingError. +func (e FundingError) String() string { + switch e { + case ErrMaxPendingChannels: + return "Number of pending channels exceed maximum" + case ErrChanTooLarge: + return "channel too large" + default: + return "unknown error" + } +} + +// Error returns the human readable version of the target FundingError. +// +// NOTE: Satisfies the Error interface. +func (e FundingError) Error() string { + return e.String() +} + +// ErrorData is a set of bytes associated with a particular sent error. A +// receiving node SHOULD only print out data verbatim if the string is composed +// solely of printable ASCII characters. For reference, the printable character +// set includes byte values 32 through 127 inclusive. +type ErrorData []byte + +// Error represents a generic error bound to an exact channel. The message +// format is purposefully general in order to allow expression of a wide array +// of possible errors. Each Error message is directed at a particular open +// channel referenced by ChannelPoint. +type Error struct { + // ChanID references the active channel in which the error occurred + // within. If the ChanID is all zeros, then this error applies to the + // entire established connection. + ChanID ChannelID + + // Data is the attached error data that describes the exact failure + // which caused the error message to be sent. + Data ErrorData +} + +// NewError creates a new Error message. +func NewError() *Error { + return &Error{} +} + +// A compile time check to ensure Error implements the lnwire.Message +// interface. +var _ Message = (*Error)(nil) + +// A compile time check to ensure Error implements the lnwire.SizeableMessage +// interface. +var _ SizeableMessage = (*Error)(nil) + +// Error returns the string representation to Error. +// +// NOTE: Satisfies the error interface. +func (c *Error) Error() string { + errMsg := "non-ascii data" + if isASCII(c.Data) { + errMsg = string(c.Data) + } + + return fmt.Sprintf("chan_id=%v, err=%v", c.ChanID, errMsg) +} + +// Decode deserializes a serialized Error message stored in the passed +// io.Reader observing the specified protocol version. +// +// This is part of the lnwire.Message interface. +func (c *Error) Decode(r io.Reader, pver uint32) error { + return ReadElements(r, + &c.ChanID, + &c.Data, + ) +} + +// Encode serializes the target Error into the passed io.Writer observing the +// protocol version specified. +// +// This is part of the lnwire.Message interface. +func (c *Error) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteBytes(w, c.ChanID[:]); err != nil { + return err + } + + return WriteErrorData(w, c.Data) +} + +// MsgType returns the integer uniquely identifying an Error message on the +// wire. +// +// This is part of the lnwire.Message interface. +func (c *Error) MsgType() MessageType { + return MsgError +} + +// SerializedSize returns the serialized size of the message in bytes. +// +// This is part of the lnwire.SizeableMessage interface. +func (c *Error) SerializedSize() (uint32, error) { + return MessageSerializedSize(c) +} + +// isASCII is a helper method that checks whether all bytes in `data` would be +// printable ASCII characters if interpreted as a string. +func isASCII(data []byte) bool { + for _, c := range data { + if c < 32 || c > 126 { + return false + } + } + return true +} diff --git a/payments/db/migration1/lnwire/extra_bytes.go b/payments/db/migration1/lnwire/extra_bytes.go new file mode 100644 index 000000000..39228c101 --- /dev/null +++ b/payments/db/migration1/lnwire/extra_bytes.go @@ -0,0 +1,309 @@ +package lnwire + +import ( + "bytes" + "fmt" + "io" + + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/tlv" +) + +// ExtraOpaqueData is the set of data that was appended to this message, some +// of which we may not actually know how to iterate or parse. By holding onto +// this data, we ensure that we're able to properly validate the set of +// signatures that cover these new fields, and ensure we're able to make +// upgrades to the network in a forwards compatible manner. +type ExtraOpaqueData []byte + +// NewExtraOpaqueData creates a new ExtraOpaqueData instance from a tlv.TypeMap. +func NewExtraOpaqueData(tlvMap tlv.TypeMap) (ExtraOpaqueData, error) { + // If the tlv map is empty, we'll want to mirror the behavior of + // decoding an empty extra opaque data field (see Decode method). + if len(tlvMap) == 0 { + return make([]byte, 0), nil + } + + // Convert the TLV map into a slice of records. + records := TlvMapToRecords(tlvMap) + + // Encode the records into the extra data byte slice. + return EncodeRecords(records) +} + +// Encode attempts to encode the raw extra bytes into the passed io.Writer. +func (e *ExtraOpaqueData) Encode(w *bytes.Buffer) error { + eBytes := []byte((*e)[:]) + if err := WriteBytes(w, eBytes); err != nil { + return err + } + + return nil +} + +// Decode attempts to unpack the raw bytes encoded in the passed-in io.Reader as +// a set of extra opaque data. +func (e *ExtraOpaqueData) Decode(r io.Reader) error { + // First, we'll attempt to read a set of bytes contained within the + // passed io.Reader (if any exist). + rawBytes, err := io.ReadAll(r) + if err != nil { + return err + } + + // If we _do_ have some bytes, then we'll swap out our backing pointer. + // This ensures that any struct that embeds this type will properly + // store the bytes once this method exits. + if len(rawBytes) > 0 { + *e = rawBytes + } else { + *e = make([]byte, 0) + } + + return nil +} + +// ValidateTLV checks that the raw bytes that make up the ExtraOpaqueData +// instance are a valid TLV stream. +func (e *ExtraOpaqueData) ValidateTLV() error { + // There is nothing to validate if the ExtraOpaqueData is nil or empty. + if e == nil || len(*e) == 0 { + return nil + } + + tlvStream, err := tlv.NewStream() + if err != nil { + return err + } + + // Ensure that the TLV stream is valid by attempting to decode it. + _, err = tlvStream.DecodeWithParsedTypesP2P(bytes.NewReader(*e)) + if err != nil { + return fmt.Errorf("invalid TLV stream: %w: %v", err, *e) + } + + return nil +} + +// PackRecords attempts to encode the set of tlv records into the target +// ExtraOpaqueData instance. The records will be encoded as a raw TLV stream +// and stored within the backing slice pointer. +func (e *ExtraOpaqueData) PackRecords( + recordProducers ...tlv.RecordProducer) error { + + // Assemble all the records passed in series, then encode them. + records := ProduceRecordsSorted(recordProducers...) + encoded, err := EncodeRecords(records) + if err != nil { + return err + } + + *e = encoded + + return nil +} + +// ExtractRecords attempts to decode any types in the internal raw bytes as if +// it were a tlv stream. The set of raw parsed types is returned, and any +// passed records (if found in the stream) will be parsed into the proper +// tlv.Record. +func (e *ExtraOpaqueData) ExtractRecords( + recordProducers ...tlv.RecordProducer) (tlv.TypeMap, error) { + + // First, assemble all the records passed in series. + records := ProduceRecordsSorted(recordProducers...) + extraBytesReader := bytes.NewReader(*e) + + // Since ExtraOpaqueData is provided by a potentially malicious peer, + // pass it into the P2P decoding variant. + return DecodeRecordsP2P(extraBytesReader, records...) +} + +// RecordProducers parses ExtraOpaqueData into a slice of TLV record producers +// by interpreting it as a TLV map. +func (e *ExtraOpaqueData) RecordProducers() ([]tlv.RecordProducer, error) { + var recordProducers []tlv.RecordProducer + + // If the instance is nil or empty, return an empty slice. + if e == nil || len(*e) == 0 { + return recordProducers, nil + } + + // Parse the extra opaque data as a TLV map. + tlvMap, err := e.ExtractRecords() + if err != nil { + return nil, err + } + + // Convert the TLV map into a slice of record producers. + records := TlvMapToRecords(tlvMap) + + return RecordsAsProducers(records), nil +} + +// EncodeMessageExtraData encodes the given recordProducers into the given +// extraData. +func EncodeMessageExtraData(extraData *ExtraOpaqueData, + recordProducers ...tlv.RecordProducer) error { + + // Treat extraData as a mutable reference. + if extraData == nil { + return fmt.Errorf("extra data cannot be nil") + } + + // Pack in the series of TLV records into this message. The order we + // pass them in doesn't matter, as the method will ensure that things + // are all properly sorted. + return extraData.PackRecords(recordProducers...) +} + +// ParseAndExtractCustomRecords parses the given extra data into the passed-in +// records, then returns any remaining records split into custom records and +// extra data. +func ParseAndExtractCustomRecords(allExtraData ExtraOpaqueData, + knownRecords ...tlv.RecordProducer) (CustomRecords, + fn.Set[tlv.Type], ExtraOpaqueData, error) { + + extraDataTlvMap, err := allExtraData.ExtractRecords(knownRecords...) + if err != nil { + return nil, nil, nil, err + } + + // Remove the known and now extracted records from the leftover extra + // data map. + parsedKnownRecords := make(fn.Set[tlv.Type], len(knownRecords)) + for _, producer := range knownRecords { + r := producer.Record() + + // Only remove the records if it was parsed (remainder is nil). + // We'll just store the type so we can tell the caller which + // records were actually parsed fully. + val, ok := extraDataTlvMap[r.Type()] + if ok && val == nil { + parsedKnownRecords.Add(r.Type()) + delete(extraDataTlvMap, r.Type()) + } + } + + // Any records from the extra data TLV map which are in the custom + // records TLV type range will be included in the custom records field + // and removed from the extra data field. + customRecordsTlvMap := make(tlv.TypeMap, len(extraDataTlvMap)) + for k, v := range extraDataTlvMap { + // Skip records that are not in the custom records TLV type + // range. + if k < MinCustomRecordsTlvType { + continue + } + + // Include the record in the custom records map. + customRecordsTlvMap[k] = v + + // Now that the record is included in the custom records map, + // we can remove it from the extra data TLV map. + delete(extraDataTlvMap, k) + } + + // Set the custom records field to the custom records specific TLV + // record map. + customRecords, err := NewCustomRecords(customRecordsTlvMap) + if err != nil { + return nil, nil, nil, err + } + + // Encode the remaining records back into the extra data field. These + // records are not in the custom records TLV type range and do not + // have associated fields in the struct that produced the records. + extraData, err := NewExtraOpaqueData(extraDataTlvMap) + if err != nil { + return nil, nil, nil, err + } + + // Help with unit testing where we might have the empty value (nil) for + // the extra data instead of the default that's returned by the + // constructor (empty slice). + if len(extraData) == 0 { + extraData = nil + } + + return customRecords, parsedKnownRecords, extraData, nil +} + +// MergeAndEncode merges the known records with the extra data and custom +// records, then encodes the merged records into raw bytes. +func MergeAndEncode(knownRecords []tlv.RecordProducer, + extraData ExtraOpaqueData, customRecords CustomRecords) ([]byte, + error) { + + // Construct a slice of all the records that we should include in the + // message extra data field. We will start by including any records from + // the extra data field. + mergedRecords, err := extraData.RecordProducers() + if err != nil { + return nil, err + } + + // Merge the known and extra data records. + mergedRecords = append(mergedRecords, knownRecords...) + + // Include custom records in the extra data wire field if they are + // present. Ensure that the custom records are validated before encoding + // them. + if err := customRecords.Validate(); err != nil { + return nil, fmt.Errorf("custom records validation error: %w", + err) + } + + // Extend the message extra data records slice with TLV records from the + // custom records field. + mergedRecords = append( + mergedRecords, customRecords.RecordProducers()..., + ) + + // Now we can sort the records and make sure there are no records with + // the same type that would collide when encoding. + sortedRecords := ProduceRecordsSorted(mergedRecords...) + if err := AssertUniqueTypes(sortedRecords); err != nil { + return nil, err + } + + return EncodeRecords(sortedRecords) +} + +// ParseAndExtractExtraData parses the given extra data into the passed-in +// records, then returns any remaining records as extra data. +func ParseAndExtractExtraData(allTlvData ExtraOpaqueData, + knownRecords ...tlv.RecordProducer) (fn.Set[tlv.Type], + ExtraOpaqueData, error) { + + extraDataTlvMap, err := allTlvData.ExtractRecords(knownRecords...) + if err != nil { + return nil, nil, err + } + + // Remove the known and now extracted records from the leftover extra + // data map. + parsedKnownRecords := make(fn.Set[tlv.Type], len(knownRecords)) + for _, producer := range knownRecords { + r := producer.Record() + + // Only remove the records if it was parsed (remainder is nil). + // We'll just store the type so we can tell the caller which + // records were actually parsed fully. + val, ok := extraDataTlvMap[r.Type()] + if ok && val == nil { + parsedKnownRecords.Add(r.Type()) + delete(extraDataTlvMap, r.Type()) + } + } + + // Encode the remaining records back into the extra data field. These + // records are not in the custom records TLV type range and do not + // have associated fields in the struct that produced the records. + extraData, err := NewExtraOpaqueData(extraDataTlvMap) + if err != nil { + return nil, nil, err + } + + return parsedKnownRecords, extraData, nil +} diff --git a/payments/db/migration1/lnwire/features.go b/payments/db/migration1/lnwire/features.go new file mode 100644 index 000000000..4e927e170 --- /dev/null +++ b/payments/db/migration1/lnwire/features.go @@ -0,0 +1,898 @@ +package lnwire + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/lightningnetwork/lnd/tlv" +) + +var ( + // ErrFeaturePairExists signals an error in feature vector construction + // where the opposing bit in a feature pair has already been set. + ErrFeaturePairExists = errors.New("feature pair exists") + + // ErrFeatureStandard is returned when attempts to modify LND's known + // set of features are made. + ErrFeatureStandard = errors.New("feature is used in standard " + + "protocol set") + + // ErrFeatureBitMaximum is returned when a feature bit exceeds the + // maximum allowable value. + ErrFeatureBitMaximum = errors.New("feature bit exceeds allowed maximum") +) + +// FeatureBit represents a feature that can be enabled in either a local or +// global feature vector at a specific bit position. Feature bits follow the +// "it's OK to be odd" rule, where features at even bit positions must be known +// to a node receiving them from a peer while odd bits do not. In accordance, +// feature bits are usually assigned in pairs, first being assigned an odd bit +// position which may later be changed to the preceding even position once +// knowledge of the feature becomes required on the network. +type FeatureBit uint16 + +const ( + // DataLossProtectRequired is a feature bit that indicates that a peer + // *requires* the other party know about the data-loss-protect optional + // feature. If the remote peer does not know of such a feature, then + // the sending peer SHOULD disconnect them. The data-loss-protect + // feature allows a peer that's lost partial data to recover their + // settled funds of the latest commitment state. + DataLossProtectRequired FeatureBit = 0 + + // DataLossProtectOptional is an optional feature bit that indicates + // that the sending peer knows of this new feature and can activate it + // it. The data-loss-protect feature allows a peer that's lost partial + // data to recover their settled funds of the latest commitment state. + DataLossProtectOptional FeatureBit = 1 + + // InitialRoutingSync is a local feature bit meaning that the receiving + // node should send a complete dump of routing information when a new + // connection is established. + InitialRoutingSync FeatureBit = 3 + + // UpfrontShutdownScriptRequired is a feature bit which indicates that a + // peer *requires* that the remote peer accept an upfront shutdown script to + // which payout is enforced on cooperative closes. + UpfrontShutdownScriptRequired FeatureBit = 4 + + // UpfrontShutdownScriptOptional is an optional feature bit which indicates + // that the peer will accept an upfront shutdown script to which payout is + // enforced on cooperative closes. + UpfrontShutdownScriptOptional FeatureBit = 5 + + // GossipQueriesRequired is a feature bit that indicates that the + // receiving peer MUST know of the set of features that allows nodes to + // more efficiently query the network view of peers on the network for + // reconciliation purposes. + GossipQueriesRequired FeatureBit = 6 + + // GossipQueriesOptional is an optional feature bit that signals that + // the setting peer knows of the set of features that allows more + // efficient network view reconciliation. + GossipQueriesOptional FeatureBit = 7 + + // TLVOnionPayloadRequired is a feature bit that indicates a node is + // able to decode the new TLV information included in the onion packet. + TLVOnionPayloadRequired FeatureBit = 8 + + // TLVOnionPayloadOptional is an optional feature bit that indicates a + // node is able to decode the new TLV information included in the onion + // packet. + TLVOnionPayloadOptional FeatureBit = 9 + + // StaticRemoteKeyRequired is a required feature bit that signals that + // within one's commitment transaction, the key used for the remote + // party's non-delay output should not be tweaked. + StaticRemoteKeyRequired FeatureBit = 12 + + // StaticRemoteKeyOptional is an optional feature bit that signals that + // within one's commitment transaction, the key used for the remote + // party's non-delay output should not be tweaked. + StaticRemoteKeyOptional FeatureBit = 13 + + // PaymentAddrRequired is a required feature bit that signals that a + // node requires payment addresses, which are used to mitigate probing + // attacks on the receiver of a payment. + PaymentAddrRequired FeatureBit = 14 + + // PaymentAddrOptional is an optional feature bit that signals that a + // node supports payment addresses, which are used to mitigate probing + // attacks on the receiver of a payment. + PaymentAddrOptional FeatureBit = 15 + + // MPPRequired is a required feature bit that signals that the receiver + // of a payment requires settlement of an invoice with more than one + // HTLC. + MPPRequired FeatureBit = 16 + + // MPPOptional is an optional feature bit that signals that the receiver + // of a payment supports settlement of an invoice with more than one + // HTLC. + MPPOptional FeatureBit = 17 + + // WumboChannelsRequired is a required feature bit that signals that a + // node is willing to accept channels larger than 2^24 satoshis. + WumboChannelsRequired FeatureBit = 18 + + // WumboChannelsOptional is an optional feature bit that signals that a + // node is willing to accept channels larger than 2^24 satoshis. + WumboChannelsOptional FeatureBit = 19 + + // AnchorsRequired is a required feature bit that signals that the node + // requires channels to be made using commitments having anchor + // outputs. + AnchorsRequired FeatureBit = 20 + + // AnchorsOptional is an optional feature bit that signals that the + // node supports channels to be made using commitments having anchor + // outputs. + AnchorsOptional FeatureBit = 21 + + // AnchorsZeroFeeHtlcTxRequired is a required feature bit that signals + // that the node requires channels having zero-fee second-level HTLC + // transactions, which also imply anchor commitments. + AnchorsZeroFeeHtlcTxRequired FeatureBit = 22 + + // AnchorsZeroFeeHtlcTxOptional is an optional feature bit that signals + // that the node supports channels having zero-fee second-level HTLC + // transactions, which also imply anchor commitments. + AnchorsZeroFeeHtlcTxOptional FeatureBit = 23 + + // RouteBlindingRequired is a required feature bit that signals that + // the node supports blinded payments. + RouteBlindingRequired FeatureBit = 24 + + // RouteBlindingOptional is an optional feature bit that signals that + // the node supports blinded payments. + RouteBlindingOptional FeatureBit = 25 + + // ShutdownAnySegwitRequired is an required feature bit that signals + // that the sender is able to properly handle/parse segwit witness + // programs up to version 16. This enables utilization of Taproot + // addresses for cooperative closure addresses. + ShutdownAnySegwitRequired FeatureBit = 26 + + // ShutdownAnySegwitOptional is an optional feature bit that signals + // that the sender is able to properly handle/parse segwit witness + // programs up to version 16. This enables utilization of Taproot + // addresses for cooperative closure addresses. + ShutdownAnySegwitOptional FeatureBit = 27 + + // AMPRequired is a required feature bit that signals that the receiver + // of a payment supports accepts spontaneous payments, i.e. + // sender-generated preimages according to BOLT XX. + AMPRequired FeatureBit = 30 + + // AMPOptional is an optional feature bit that signals that the receiver + // of a payment supports accepts spontaneous payments, i.e. + // sender-generated preimages according to BOLT XX. + AMPOptional FeatureBit = 31 + + // QuiescenceRequired is a required feature bit that denotes that a + // connection established with this node must support the quiescence + // protocol if it wants to have a channel relationship. + QuiescenceRequired FeatureBit = 34 + + // QuiescenceOptional is an optional feature bit that denotes that a + // connection established with this node is permitted to use the + // quiescence protocol. + QuiescenceOptional FeatureBit = 35 + + // ExplicitChannelTypeRequired is a required bit that denotes that a + // connection established with this node is to use explicit channel + // commitment types for negotiation instead of the existing implicit + // negotiation methods. With this bit, there is no longer a "default" + // implicit channel commitment type, allowing a connection to + // open/maintain types of several channels over its lifetime. + ExplicitChannelTypeRequired = 44 + + // ExplicitChannelTypeOptional is an optional bit that denotes that a + // connection established with this node is to use explicit channel + // commitment types for negotiation instead of the existing implicit + // negotiation methods. With this bit, there is no longer a "default" + // implicit channel commitment type, allowing a connection to + // TODO: Decide on actual feature bit value. + ExplicitChannelTypeOptional = 45 + + // ScidAliasRequired is a required feature bit that signals that the + // node requires understanding of ShortChannelID aliases in the TLV + // segment of the channel_ready message. + ScidAliasRequired FeatureBit = 46 + + // ScidAliasOptional is an optional feature bit that signals that the + // node understands ShortChannelID aliases in the TLV segment of the + // channel_ready message. + ScidAliasOptional FeatureBit = 47 + + // PaymentMetadataRequired is a required bit that denotes that if an + // invoice contains metadata, it must be passed along with the payment + // htlc(s). + PaymentMetadataRequired = 48 + + // PaymentMetadataOptional is an optional bit that denotes that if an + // invoice contains metadata, it may be passed along with the payment + // htlc(s). + PaymentMetadataOptional = 49 + + // ZeroConfRequired is a required feature bit that signals that the + // node requires understanding of the zero-conf channel_type. + ZeroConfRequired FeatureBit = 50 + + // ZeroConfOptional is an optional feature bit that signals that the + // node understands the zero-conf channel type. + ZeroConfOptional FeatureBit = 51 + + // KeysendRequired is a required bit that indicates that the node is + // able and willing to accept keysend payments. + KeysendRequired = 54 + + // KeysendOptional is an optional bit that indicates that the node is + // able and willing to accept keysend payments. + KeysendOptional = 55 + + // RbfCoopCloseRequired is a required feature bit that signals that + // the new RBF-based co-op close protocol is supported. + RbfCoopCloseRequired = 60 + + // RbfCoopCloseOptional is an optional feature bit that signals that the + // new RBF-based co-op close protocol is supported. + RbfCoopCloseOptional = 61 + + // RbfCoopCloseRequiredStaging is a required feature bit that signals + // that the new RBF-based co-op close protocol is supported. + RbfCoopCloseRequiredStaging = 160 + + // RbfCoopCloseOptionalStaging is an optional feature bit that signals + // that the new RBF-based co-op close protocol is supported. + RbfCoopCloseOptionalStaging = 161 + + // ScriptEnforcedLeaseRequired is a required feature bit that signals + // that the node requires channels having zero-fee second-level HTLC + // transactions, which also imply anchor commitments, along with an + // additional CLTV constraint of a channel lease's expiration height + // applied to all outputs that pay directly to the channel initiator. + // + // TODO: Decide on actual feature bit value. + ScriptEnforcedLeaseRequired FeatureBit = 2022 + + // ScriptEnforcedLeaseOptional is an optional feature bit that signals + // that the node requires channels having zero-fee second-level HTLC + // transactions, which also imply anchor commitments, along with an + // additional CLTV constraint of a channel lease's expiration height + // applied to all outputs that pay directly to the channel initiator. + // + // TODO: Decide on actual feature bit value. + ScriptEnforcedLeaseOptional FeatureBit = 2023 + + // SimpleTaprootChannelsRequiredFinal is a required bit that indicates + // the node is able to create taproot-native channels. This is the + // final feature bit to be used once the channel type is finalized. + SimpleTaprootChannelsRequiredFinal = 80 + + // SimpleTaprootChannelsOptionalFinal is an optional bit that indicates + // the node is able to create taproot-native channels. This is the + // final feature bit to be used once the channel type is finalized. + SimpleTaprootChannelsOptionalFinal = 81 + + // SimpleTaprootChannelsRequiredStaging is a required bit that indicates + // the node is able to create taproot-native channels. This is a + // feature bit used in the wild while the channel type is still being + // finalized. + SimpleTaprootChannelsRequiredStaging = 180 + + // SimpleTaprootChannelsOptionalStaging is an optional bit that + // indicates the node is able to create taproot-native channels. This + // is a feature bit used in the wild while the channel type is still + // being finalized. + SimpleTaprootChannelsOptionalStaging = 181 + + // ExperimentalAccountabilityRequired is a required feature bit that + // indicates that the node will relay experimental accountability + // signals. + ExperimentalAccountabilityRequired FeatureBit = 260 + + // ExperimentalAccountabilityOptional is an optional feature bit that + // indicates that the node will relay experimental accountability + // signals. + ExperimentalAccountabilityOptional FeatureBit = 261 + + // Bolt11BlindedPathsRequired is a required feature bit that indicates + // that the node is able to understand the blinded path tagged field in + // a BOLT 11 invoice. + Bolt11BlindedPathsRequired = 262 + + // Bolt11BlindedPathsOptional is an optional feature bit that indicates + // that the node is able to understand the blinded path tagged field in + // a BOLT 11 invoice. + Bolt11BlindedPathsOptional = 263 + + // SimpleTaprootOverlayChansRequired is a required bit that indicates + // support for the special custom taproot overlay channel. + SimpleTaprootOverlayChansOptional = 2025 + + // SimpleTaprootOverlayChansRequired is a required bit that indicates + // support for the special custom taproot overlay channel. + SimpleTaprootOverlayChansRequired = 2026 + + // MaxBolt11Feature is the maximum feature bit value allowed in bolt 11 + // invoices. + // + // The base 32 encoded tagged fields in invoices are limited to 10 bits + // to express the length of the field's data. + //nolint:ll + // See: https://github.com/lightning/bolts/blob/master/11-payment-encoding.md#tagged-fields + // + // With a maximum length field of 1023 (2^10 -1) and 5 bit encoding, + // the highest feature bit that can be expressed is: + // 1023 * 5 - 1 = 5114. + MaxBolt11Feature = 5114 +) + +// IsRequired returns true if the feature bit is even, and false otherwise. +func (b FeatureBit) IsRequired() bool { + return b&0x01 == 0x00 +} + +// Features is a mapping of known feature bits to a descriptive name. All known +// feature bits must be assigned a name in this mapping, and feature bit pairs +// must be assigned together for correct behavior. +var Features = map[FeatureBit]string{ + DataLossProtectRequired: "data-loss-protect", + DataLossProtectOptional: "data-loss-protect", + InitialRoutingSync: "initial-routing-sync", + UpfrontShutdownScriptRequired: "upfront-shutdown-script", + UpfrontShutdownScriptOptional: "upfront-shutdown-script", + GossipQueriesRequired: "gossip-queries", + GossipQueriesOptional: "gossip-queries", + TLVOnionPayloadRequired: "tlv-onion", + TLVOnionPayloadOptional: "tlv-onion", + StaticRemoteKeyOptional: "static-remote-key", + StaticRemoteKeyRequired: "static-remote-key", + PaymentAddrOptional: "payment-addr", + PaymentAddrRequired: "payment-addr", + MPPOptional: "multi-path-payments", + MPPRequired: "multi-path-payments", + AnchorsRequired: "anchor-commitments", + AnchorsOptional: "anchor-commitments", + AnchorsZeroFeeHtlcTxRequired: "anchors-zero-fee-htlc-tx", + AnchorsZeroFeeHtlcTxOptional: "anchors-zero-fee-htlc-tx", + WumboChannelsRequired: "wumbo-channels", + WumboChannelsOptional: "wumbo-channels", + AMPRequired: "amp", + AMPOptional: "amp", + QuiescenceRequired: "quiescence", + QuiescenceOptional: "quiescence", + PaymentMetadataOptional: "payment-metadata", + PaymentMetadataRequired: "payment-metadata", + ExplicitChannelTypeOptional: "explicit-commitment-type", + ExplicitChannelTypeRequired: "explicit-commitment-type", + KeysendOptional: "keysend", + KeysendRequired: "keysend", + ScriptEnforcedLeaseRequired: "script-enforced-lease", + ScriptEnforcedLeaseOptional: "script-enforced-lease", + ScidAliasRequired: "scid-alias", + ScidAliasOptional: "scid-alias", + ZeroConfRequired: "zero-conf", + ZeroConfOptional: "zero-conf", + RouteBlindingRequired: "route-blinding", + RouteBlindingOptional: "route-blinding", + ShutdownAnySegwitRequired: "shutdown-any-segwit", + ShutdownAnySegwitOptional: "shutdown-any-segwit", + SimpleTaprootChannelsRequiredFinal: "simple-taproot-chans", + SimpleTaprootChannelsOptionalFinal: "simple-taproot-chans", + SimpleTaprootChannelsRequiredStaging: "simple-taproot-chans-x", + SimpleTaprootChannelsOptionalStaging: "simple-taproot-chans-x", + SimpleTaprootOverlayChansOptional: "taproot-overlay-chans", + SimpleTaprootOverlayChansRequired: "taproot-overlay-chans", + ExperimentalAccountabilityRequired: "accountable-x", + ExperimentalAccountabilityOptional: "accountable-x", + Bolt11BlindedPathsOptional: "bolt-11-blinded-paths", + Bolt11BlindedPathsRequired: "bolt-11-blinded-paths", + RbfCoopCloseOptional: "rbf-coop-close", + RbfCoopCloseRequired: "rbf-coop-close", + RbfCoopCloseOptionalStaging: "rbf-coop-close-x", + RbfCoopCloseRequiredStaging: "rbf-coop-close-x", +} + +// RawFeatureVector represents a set of feature bits as defined in BOLT-09. A +// RawFeatureVector itself just stores a set of bit flags but can be used to +// construct a FeatureVector which binds meaning to each bit. Feature vectors +// can be serialized and deserialized to/from a byte representation that is +// transmitted in Lightning network messages. +type RawFeatureVector struct { + features map[FeatureBit]struct{} +} + +// NewRawFeatureVector creates a feature vector with all of the feature bits +// given as arguments enabled. +func NewRawFeatureVector(bits ...FeatureBit) *RawFeatureVector { + fv := &RawFeatureVector{features: make(map[FeatureBit]struct{})} + for _, bit := range bits { + fv.Set(bit) + } + return fv +} + +// IsEmpty returns whether the feature vector contains any feature bits. +func (fv RawFeatureVector) IsEmpty() bool { + return len(fv.features) == 0 +} + +// OnlyContains determines whether only the specified feature bits are found. +func (fv RawFeatureVector) OnlyContains(bits ...FeatureBit) bool { + if len(bits) != len(fv.features) { + return false + } + for _, bit := range bits { + if !fv.IsSet(bit) { + return false + } + } + return true +} + +// Equals determines whether two features vectors contain exactly the same +// features. +func (fv RawFeatureVector) Equals(other *RawFeatureVector) bool { + if len(fv.features) != len(other.features) { + return false + } + for bit := range fv.features { + if _, ok := other.features[bit]; !ok { + return false + } + } + return true +} + +// Merge sets all feature bits in other on the receiver's feature vector. +func (fv *RawFeatureVector) Merge(other *RawFeatureVector) error { + for bit := range other.features { + err := fv.SafeSet(bit) + if err != nil { + return err + } + } + return nil +} + +// ValidateUpdate checks whether a feature vector can safely be updated to the +// new feature vector provided, checking that it does not alter any of the +// "standard" features that are defined by LND. The new feature vector should +// be inclusive of all features in the original vector that it still wants to +// advertise, setting and unsetting updates as desired. Features in the vector +// are also checked against a maximum inclusive value, as feature vectors in +// different contexts have different maximum values. +func (fv *RawFeatureVector) ValidateUpdate(other *RawFeatureVector, + maximumValue FeatureBit) error { + + // Run through the new set of features and check that we're not adding + // any feature bits that are defined but not set in LND. + for feature := range other.features { + if fv.IsSet(feature) { + continue + } + + if feature > maximumValue { + return fmt.Errorf("can't set feature bit %d: %w %v", + feature, ErrFeatureBitMaximum, + maximumValue) + } + + if name, known := Features[feature]; known { + return fmt.Errorf("can't set feature "+ + "bit %d (%v): %w", feature, name, + ErrFeatureStandard) + } + } + + // Check that the new feature vector for this set does not unset any + // features that are standard in LND by comparing the features in our + // current set to the omitted values in the new set. + for feature := range fv.features { + if other.IsSet(feature) { + continue + } + + if name, known := Features[feature]; known { + return fmt.Errorf("can't unset feature "+ + "bit %d (%v): %w", feature, name, + ErrFeatureStandard) + } + } + + return nil +} + +// ValidatePairs checks each feature bit in a raw vector to ensure that the +// opposing bit is not set, validating that the vector has either the optional +// or required bit set, not both. +func (fv *RawFeatureVector) ValidatePairs() error { + for feature := range fv.features { + if _, ok := fv.features[feature^1]; ok { + return ErrFeaturePairExists + } + } + + return nil +} + +// Clone makes a copy of a feature vector. +func (fv *RawFeatureVector) Clone() *RawFeatureVector { + newFeatures := NewRawFeatureVector() + for bit := range fv.features { + newFeatures.Set(bit) + } + return newFeatures +} + +// IsSet returns whether a particular feature bit is enabled in the vector. +func (fv *RawFeatureVector) IsSet(feature FeatureBit) bool { + _, ok := fv.features[feature] + return ok +} + +// Set marks a feature as enabled in the vector. +func (fv *RawFeatureVector) Set(feature FeatureBit) { + fv.features[feature] = struct{}{} +} + +// SafeSet sets the chosen feature bit in the feature vector, but returns an +// error if the opposing feature bit is already set. This ensures both that we +// are creating properly structured feature vectors, and in some cases, that +// peers are sending properly encoded ones, i.e. it can't be both optional and +// required. +func (fv *RawFeatureVector) SafeSet(feature FeatureBit) error { + if _, ok := fv.features[feature^1]; ok { + return ErrFeaturePairExists + } + + fv.Set(feature) + return nil +} + +// Unset marks a feature as disabled in the vector. +func (fv *RawFeatureVector) Unset(feature FeatureBit) { + delete(fv.features, feature) +} + +// SerializeSize returns the number of bytes needed to represent feature vector +// in byte format. +func (fv *RawFeatureVector) SerializeSize() int { + // We calculate byte-length via the largest bit index. + return fv.serializeSize(8) +} + +// SerializeSize32 returns the number of bytes needed to represent feature +// vector in base32 format. +func (fv *RawFeatureVector) SerializeSize32() int { + // We calculate base32-length via the largest bit index. + return fv.serializeSize(5) +} + +// serializeSize returns the number of bytes required to encode the feature +// vector using at most width bits per encoded byte. +func (fv *RawFeatureVector) serializeSize(width int) int { + // Find the largest feature bit index + max := -1 + for feature := range fv.features { + index := int(feature) + if index > max { + max = index + } + } + if max == -1 { + return 0 + } + + return max/width + 1 +} + +// Encode writes the feature vector in byte representation. Every feature +// encoded as a bit, and the bit vector is serialized using the least number of +// bytes. Since the bit vector length is variable, the first two bytes of the +// serialization represent the length. +func (fv *RawFeatureVector) Encode(w io.Writer) error { + // Write length of feature vector. + var l [2]byte + length := fv.SerializeSize() + binary.BigEndian.PutUint16(l[:], uint16(length)) + if _, err := w.Write(l[:]); err != nil { + return err + } + + return fv.encode(w, length, 8) +} + +// EncodeBase256 writes the feature vector in base256 representation. Every +// feature is encoded as a bit, and the bit vector is serialized using the least +// number of bytes. +func (fv *RawFeatureVector) EncodeBase256(w io.Writer) error { + length := fv.SerializeSize() + return fv.encode(w, length, 8) +} + +// EncodeBase32 writes the feature vector in base32 representation. Every feature +// is encoded as a bit, and the bit vector is serialized using the least number of +// bytes. +func (fv *RawFeatureVector) EncodeBase32(w io.Writer) error { + length := fv.SerializeSize32() + return fv.encode(w, length, 5) +} + +// encode writes the feature vector +func (fv *RawFeatureVector) encode(w io.Writer, length, width int) error { + // Generate the data and write it. + data := make([]byte, length) + for feature := range fv.features { + byteIndex := int(feature) / width + bitIndex := int(feature) % width + data[length-byteIndex-1] |= 1 << uint(bitIndex) + } + + _, err := w.Write(data) + return err +} + +// Decode reads the feature vector from its byte representation. Every feature +// is encoded as a bit, and the bit vector is serialized using the least number +// of bytes. Since the bit vector length is variable, the first two bytes of the +// serialization represent the length. +func (fv *RawFeatureVector) Decode(r io.Reader) error { + // Read the length of the feature vector. + var l [2]byte + if _, err := io.ReadFull(r, l[:]); err != nil { + return err + } + length := binary.BigEndian.Uint16(l[:]) + + return fv.decode(r, int(length), 8) +} + +// DecodeBase256 reads the feature vector from its base256 representation. Every +// feature encoded as a bit, and the bit vector is serialized using the least +// number of bytes. +func (fv *RawFeatureVector) DecodeBase256(r io.Reader, length int) error { + return fv.decode(r, length, 8) +} + +// DecodeBase32 reads the feature vector from its base32 representation. Every +// feature encoded as a bit, and the bit vector is serialized using the least +// number of bytes. +func (fv *RawFeatureVector) DecodeBase32(r io.Reader, length int) error { + return fv.decode(r, length, 5) +} + +// decode reads a feature vector from the next length bytes of the io.Reader, +// assuming each byte has width feature bits encoded per byte. +func (fv *RawFeatureVector) decode(r io.Reader, length, width int) error { + // Read the feature vector data. + data := make([]byte, length) + if _, err := io.ReadFull(r, data); err != nil { + return err + } + + // Set feature bits from parsed data. + bitsNumber := len(data) * width + for i := 0; i < bitsNumber; i++ { + byteIndex := int(i / width) + bitIndex := uint(i % width) + if (data[length-byteIndex-1]>>bitIndex)&1 == 1 { + fv.Set(FeatureBit(i)) + } + } + + return nil +} + +// sizeFunc returns the length required to encode the feature vector. +func (fv *RawFeatureVector) sizeFunc() uint64 { + return uint64(fv.SerializeSize()) +} + +// Record returns a TLV record that can be used to encode/decode raw feature +// vectors. Note that the length of the feature vector is not included, because +// it is covered by the TLV record's length field. +func (fv *RawFeatureVector) Record() tlv.Record { + return tlv.MakeDynamicRecord( + 0, fv, fv.sizeFunc, rawFeatureEncoder, rawFeatureDecoder, + ) +} + +// rawFeatureEncoder is a custom TLV encoder for raw feature vectors. +func rawFeatureEncoder(w io.Writer, val interface{}, _ *[8]byte) error { + if v, ok := val.(*RawFeatureVector); ok { + // Encode the feature bits as a byte slice without its length + // prepended, as that's already taken care of by the TLV record. + fv := *v + return fv.encode(w, fv.SerializeSize(), 8) + } + + return tlv.NewTypeForEncodingErr(val, "lnwire.RawFeatureVector") +} + +// rawFeatureDecoder is a custom TLV decoder for raw feature vectors. +func rawFeatureDecoder(r io.Reader, val interface{}, _ *[8]byte, + l uint64) error { + + if v, ok := val.(*RawFeatureVector); ok { + fv := NewRawFeatureVector() + if err := fv.decode(r, int(l), 8); err != nil { + return err + } + *v = *fv + + return nil + } + + return tlv.NewTypeForEncodingErr(val, "lnwire.RawFeatureVector") +} + +// FeatureVector represents a set of enabled features. The set stores +// information on enabled flags and metadata about the feature names. A feature +// vector is serializable to a compact byte representation that is included in +// Lightning network messages. +type FeatureVector struct { + *RawFeatureVector + featureNames map[FeatureBit]string +} + +// NewFeatureVector constructs a new FeatureVector from a raw feature vector +// and mapping of feature definitions. If the feature vector argument is nil, a +// new one will be constructed with no enabled features. +func NewFeatureVector(featureVector *RawFeatureVector, + featureNames map[FeatureBit]string) *FeatureVector { + + if featureVector == nil { + featureVector = NewRawFeatureVector() + } + return &FeatureVector{ + RawFeatureVector: featureVector, + featureNames: featureNames, + } +} + +// EmptyFeatureVector returns a feature vector with no bits set. +func EmptyFeatureVector() *FeatureVector { + return NewFeatureVector(nil, Features) +} + +// Record implements the RecordProducer interface for FeatureVector. Note that +// it uses a zero-value type is used to produce the record, as we expect this +// type value to be overwritten when used in generic TLV record production. +// This allows a single Record function to serve in the many different contexts +// in which feature vectors are encoded. This record wraps the encoding/ +// decoding for our raw feature vectors so that we can directly parse fully +// formed feature vector types. +func (fv *FeatureVector) Record() tlv.Record { + return tlv.MakeDynamicRecord(0, fv, fv.sizeFunc, + func(w io.Writer, val interface{}, buf *[8]byte) error { + if f, ok := val.(*FeatureVector); ok { + return rawFeatureEncoder( + w, f.RawFeatureVector, buf, + ) + } + + return tlv.NewTypeForEncodingErr( + val, "*lnwire.FeatureVector", + ) + }, + func(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if f, ok := val.(*FeatureVector); ok { + features := NewFeatureVector(nil, Features) + err := rawFeatureDecoder( + r, features.RawFeatureVector, buf, l, + ) + if err != nil { + return err + } + + *f = *features + + return nil + } + + return tlv.NewTypeForDecodingErr( + val, "*lnwire.FeatureVector", l, l, + ) + }, + ) +} + +// HasFeature returns whether a particular feature is included in the set. The +// feature can be seen as set either if the bit is set directly OR the queried +// bit has the same meaning as its corresponding even/odd bit, which is set +// instead. The second case is because feature bits are generally assigned in +// pairs where both the even and odd position represent the same feature. +func (fv *FeatureVector) HasFeature(feature FeatureBit) bool { + return fv.IsSet(feature) || + (fv.isFeatureBitPair(feature) && fv.IsSet(feature^1)) +} + +// RequiresFeature returns true if the referenced feature vector *requires* +// that the given required bit be set. This method can be used with both +// optional and required feature bits as a parameter. +func (fv *FeatureVector) RequiresFeature(feature FeatureBit) bool { + // If we weren't passed a required feature bit, then we'll flip the + // lowest bit to query for the required version of the feature. This + // lets callers pass in both the optional and required bits. + if !feature.IsRequired() { + feature ^= 1 + } + + return fv.IsSet(feature) +} + +// UnknownRequiredFeatures returns a list of feature bits set in the vector +// that are unknown and in an even bit position. Feature bits with an even +// index must be known to a node receiving the feature vector in a message. +func (fv *FeatureVector) UnknownRequiredFeatures() []FeatureBit { + var unknown []FeatureBit + for feature := range fv.features { + if feature%2 == 0 && !fv.IsKnown(feature) { + unknown = append(unknown, feature) + } + } + return unknown +} + +// UnknownFeatures returns a boolean if a feature vector contains *any* +// unknown features (even if they are odd). +func (fv *FeatureVector) UnknownFeatures() bool { + for feature := range fv.features { + if !fv.IsKnown(feature) { + return true + } + } + + return false +} + +// Name returns a string identifier for the feature represented by this bit. If +// the bit does not represent a known feature, this returns a string indicating +// as such. +func (fv *FeatureVector) Name(bit FeatureBit) string { + name, known := fv.featureNames[bit] + if !known { + return "unknown" + } + return name +} + +// IsKnown returns whether this feature bit represents a known feature. +func (fv *FeatureVector) IsKnown(bit FeatureBit) bool { + _, known := fv.featureNames[bit] + return known +} + +// isFeatureBitPair returns whether this feature bit and its corresponding +// even/odd bit both represent the same feature. This may often be the case as +// bits are generally assigned in pairs, first being assigned an odd bit +// position then being promoted to an even bit position once the network is +// ready. +func (fv *FeatureVector) isFeatureBitPair(bit FeatureBit) bool { + name1, known1 := fv.featureNames[bit] + name2, known2 := fv.featureNames[bit^1] + return known1 && known2 && name1 == name2 +} + +// Features returns the set of raw features contained in the feature vector. +func (fv *FeatureVector) Features() map[FeatureBit]struct{} { + fs := make(map[FeatureBit]struct{}, len(fv.RawFeatureVector.features)) + for b := range fv.RawFeatureVector.features { + fs[b] = struct{}{} + } + return fs +} + +// Clone copies a feature vector, carrying over its feature bits. The feature +// names are not copied. +func (fv *FeatureVector) Clone() *FeatureVector { + features := fv.RawFeatureVector.Clone() + return NewFeatureVector(features, fv.featureNames) +} diff --git a/payments/db/migration1/lnwire/interfaces.go b/payments/db/migration1/lnwire/interfaces.go new file mode 100644 index 000000000..115b07d7b --- /dev/null +++ b/payments/db/migration1/lnwire/interfaces.go @@ -0,0 +1,174 @@ +package lnwire + +import ( + "fmt" + + "github.com/btcsuite/btcd/chaincfg/chainhash" +) + +// GossipVersion is a version number that describes the version of the +// gossip protocol that a gossip message was gossiped on. +type GossipVersion uint8 + +const ( + // GossipVersion1 is the initial version of the gossip protocol as + // defined in BOLT 7. This version of the protocol can only gossip P2WSH + // channels and makes use of ECDSA signatures. + GossipVersion1 GossipVersion = 1 + + // GossipVersion2 is the newest version of the gossip protocol. This + // version adds support for P2TR channels and makes use of Schnorr + // signatures. The BOLT number is TBD. + GossipVersion2 GossipVersion = 2 +) + +// String returns a string representation of the protocol version. +func (v GossipVersion) String() string { + return fmt.Sprintf("V%d", v) +} + +// GossipMessage is an interface that must be satisfied by all messages that are +// part of the gossip protocol. +type GossipMessage interface { + // GossipVersion returns the version of the gossip protocol that a + // message is part of. + GossipVersion() GossipVersion +} + +// AnnounceSignatures is an interface that represents a message used to +// exchange signatures of a ChannelAnnouncment message during the funding flow. +type AnnounceSignatures interface { + // SCID returns the ShortChannelID of the channel. + SCID() ShortChannelID + + // ChanID returns the ChannelID identifying the channel. + ChanID() ChannelID + + Message + GossipMessage +} + +// ChannelAnnouncement is an interface that must be satisfied by any message +// used to announce and prove the existence of a channel. +type ChannelAnnouncement interface { + // SCID returns the short channel ID of the channel. + SCID() ShortChannelID + + // GetChainHash returns the hash of the chain which this channel's + // funding transaction is confirmed in. + GetChainHash() chainhash.Hash + + // Node1KeyBytes returns the bytes representing the public key of node + // 1 in the channel. + Node1KeyBytes() [33]byte + + // Node2KeyBytes returns the bytes representing the public key of node + // 2 in the channel. + Node2KeyBytes() [33]byte + + Message + GossipMessage +} + +// CompareResult represents the result after comparing two things. +type CompareResult uint8 + +const ( + // LessThan indicates that base object is less than the object it was + // compared to. + LessThan CompareResult = iota + + // EqualTo indicates that the base object is equal to the object it was + // compared to. + EqualTo + + // GreaterThan indicates that base object is greater than the object it + // was compared to. + GreaterThan +) + +// ChannelUpdate is an interface that describes a message used to update the +// forwarding rules of a channel. +type ChannelUpdate interface { + // SCID returns the ShortChannelID of the channel that the update + // applies to. + SCID() ShortChannelID + + // IsNode1 is true if the update was produced by node 1 of the channel + // peers. Node 1 is the node with the lexicographically smaller public + // key. + IsNode1() bool + + // IsDisabled is true if the update is announcing that the channel + // should be considered disabled. + IsDisabled() bool + + // GetChainHash returns the hash of the chain that the message is + // referring to. + GetChainHash() chainhash.Hash + + // ForwardingPolicy returns the set of forwarding constraints of the + // update. + ForwardingPolicy() *ForwardingPolicy + + // CmpAge can be used to determine if the update is older or newer than + // the passed update. It returns LessThan if this update is older than + // the passed update, GreaterThan if it is newer and EqualTo if they are + // the same age. + CmpAge(update ChannelUpdate) (CompareResult, error) + + // SetDisabledFlag can be used to adjust the disabled flag of an update. + SetDisabledFlag(bool) + + // SetSCID can be used to overwrite the SCID of the update. + SetSCID(scid ShortChannelID) + + Message + GossipMessage +} + +// NodeAnnouncement is an interface that must be satisfied by any message used +// to announce the existence of a node. +type NodeAnnouncement interface { + // NodePub returns the identity public key of the node. + NodePub() [33]byte + + // NodeFeatures returns the set of features supported by the node. + NodeFeatures() *FeatureVector + + // TimestampDesc returns a human-readable description of the + // timestamp of the announcement. + TimestampDesc() string + + Message + GossipMessage +} + +// ForwardingPolicy defines the set of forwarding constraints advertised in a +// ChannelUpdate message. +type ForwardingPolicy struct { + // TimeLockDelta is the minimum number of blocks that the node requires + // to be added to the expiry of HTLCs. This is a security parameter + // determined by the node operator. This value represents the required + // gap between the time locks of the incoming and outgoing HTLC's set + // to this node. + TimeLockDelta uint16 + + // BaseFee is the base fee that must be used for incoming HTLC's to + // this particular channel. This value will be tacked onto the required + // for a payment independent of the size of the payment. + BaseFee MilliSatoshi + + // FeeRate is the fee rate that will be charged per millionth of a + // satoshi. + FeeRate MilliSatoshi + + // HtlcMinimumMsat is the minimum HTLC value which will be accepted. + MinHTLC MilliSatoshi + + // HasMaxHTLC is true if the MaxHTLC field is provided in the update. + HasMaxHTLC bool + + // HtlcMaximumMsat is the maximum HTLC value which will be accepted. + MaxHTLC MilliSatoshi +} diff --git a/payments/db/migration1/lnwire/lnwire.go b/payments/db/migration1/lnwire/lnwire.go new file mode 100644 index 000000000..09057dae4 --- /dev/null +++ b/payments/db/migration1/lnwire/lnwire.go @@ -0,0 +1,426 @@ +package lnwire + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/btcutil" +) + +const ( + // MaxSliceLength is the maximum allowed length for any opaque byte + // slices in the wire protocol. + MaxSliceLength = 65535 + + // MaxMsgBody is the largest payload any message is allowed to provide. + // This is two less than the MaxSliceLength as each message has a 2 + // byte type that precedes the message body. + MaxMsgBody = 65533 +) + +// PkScript is simple type definition which represents a raw serialized public +// key script. +type PkScript []byte + +// WriteElement is a one-stop shop to write the big endian representation of +// any element which is to be serialized for the wire protocol. +// +// TODO(yy): rm this method once we finish dereferencing it from other +// packages. +func WriteElement(w *bytes.Buffer, element interface{}) error { + switch e := element.(type) { + case uint8: + var b [1]byte + b[0] = e + if _, err := w.Write(b[:]); err != nil { + return err + } + + case uint16: + var b [2]byte + binary.BigEndian.PutUint16(b[:], e) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case ChanUpdateMsgFlags: + var b [1]byte + b[0] = uint8(e) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case ChanUpdateChanFlags: + var b [1]byte + b[0] = uint8(e) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case MilliSatoshi: + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(e)) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case btcutil.Amount: + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(e)) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case uint32: + var b [4]byte + binary.BigEndian.PutUint32(b[:], e) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case uint64: + var b [8]byte + binary.BigEndian.PutUint64(b[:], e) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case *btcec.PublicKey: + if e == nil { + return fmt.Errorf("cannot write nil pubkey") + } + + var b [33]byte + serializedPubkey := e.SerializeCompressed() + copy(b[:], serializedPubkey) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case []Sig: + var b [2]byte + numSigs := uint16(len(e)) + binary.BigEndian.PutUint16(b[:], numSigs) + if _, err := w.Write(b[:]); err != nil { + return err + } + + for _, sig := range e { + if err := WriteElement(w, sig); err != nil { + return err + } + } + + case Sig: + // Write buffer + if _, err := w.Write(e.bytes[:]); err != nil { + return err + } + + case ErrorData: + var l [2]byte + binary.BigEndian.PutUint16(l[:], uint16(len(e))) + if _, err := w.Write(l[:]); err != nil { + return err + } + + if _, err := w.Write(e[:]); err != nil { + return err + } + + case [33]byte: + if _, err := w.Write(e[:]); err != nil { + return err + } + + case []byte: + if _, err := w.Write(e[:]); err != nil { + return err + } + + case *RawFeatureVector: + if e == nil { + return fmt.Errorf("cannot write nil feature vector") + } + + if err := e.Encode(w); err != nil { + return err + } + + case ChannelID: + if _, err := w.Write(e[:]); err != nil { + return err + } + + case FailCode: + if err := WriteElement(w, uint16(e)); err != nil { + return err + } + + case ShortChannelID: + // Check that field fit in 3 bytes and write the blockHeight + if e.BlockHeight > ((1 << 24) - 1) { + return errors.New("block height should fit in 3 bytes") + } + + var blockHeight [4]byte + binary.BigEndian.PutUint32(blockHeight[:], e.BlockHeight) + + if _, err := w.Write(blockHeight[1:]); err != nil { + return err + } + + // Check that field fit in 3 bytes and write the txIndex + if e.TxIndex > ((1 << 24) - 1) { + return errors.New("tx index should fit in 3 bytes") + } + + var txIndex [4]byte + binary.BigEndian.PutUint32(txIndex[:], e.TxIndex) + if _, err := w.Write(txIndex[1:]); err != nil { + return err + } + + // Write the txPosition + var txPosition [2]byte + binary.BigEndian.PutUint16(txPosition[:], e.TxPosition) + if _, err := w.Write(txPosition[:]); err != nil { + return err + } + + case bool: + var b [1]byte + if e { + b[0] = 1 + } + if _, err := w.Write(b[:]); err != nil { + return err + } + + case ExtraOpaqueData: + return e.Encode(w) + + default: + return fmt.Errorf("unknown type in WriteElement: %T", e) + } + + return nil +} + +// WriteElements is writes each element in the elements slice to the passed +// buffer using WriteElement. +// +// TODO(yy): rm this method once we finish dereferencing it from other +// packages. +func WriteElements(buf *bytes.Buffer, elements ...interface{}) error { + for _, element := range elements { + err := WriteElement(buf, element) + if err != nil { + return err + } + } + return nil +} + +// ReadElement is a one-stop utility function to deserialize any datastructure +// encoded using the serialization format of lnwire. +func ReadElement(r io.Reader, element interface{}) error { + var err error + switch e := element.(type) { + case *bool: + var b [1]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + + if b[0] == 1 { + *e = true + } + + case *uint8: + var b [1]uint8 + if _, err := r.Read(b[:]); err != nil { + return err + } + *e = b[0] + + case *uint16: + var b [2]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + *e = binary.BigEndian.Uint16(b[:]) + + case *ChanUpdateMsgFlags: + var b [1]uint8 + if _, err := r.Read(b[:]); err != nil { + return err + } + *e = ChanUpdateMsgFlags(b[0]) + + case *ChanUpdateChanFlags: + var b [1]uint8 + if _, err := r.Read(b[:]); err != nil { + return err + } + *e = ChanUpdateChanFlags(b[0]) + + case *uint32: + var b [4]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + *e = binary.BigEndian.Uint32(b[:]) + + case *uint64: + var b [8]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + *e = binary.BigEndian.Uint64(b[:]) + + case *MilliSatoshi: + var b [8]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + *e = MilliSatoshi(int64(binary.BigEndian.Uint64(b[:]))) + + case *btcutil.Amount: + var b [8]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + *e = btcutil.Amount(int64(binary.BigEndian.Uint64(b[:]))) + + case **btcec.PublicKey: + var b [btcec.PubKeyBytesLenCompressed]byte + if _, err = io.ReadFull(r, b[:]); err != nil { + return err + } + + pubKey, err := btcec.ParsePubKey(b[:]) + if err != nil { + return err + } + *e = pubKey + + case *RawFeatureVector: + f := NewRawFeatureVector() + err = f.Decode(r) + if err != nil { + return err + } + *e = *f + + case **RawFeatureVector: + f := NewRawFeatureVector() + err = f.Decode(r) + if err != nil { + return err + } + *e = f + + case *[]Sig: + var l [2]byte + if _, err := io.ReadFull(r, l[:]); err != nil { + return err + } + numSigs := binary.BigEndian.Uint16(l[:]) + + var sigs []Sig + if numSigs > 0 { + sigs = make([]Sig, numSigs) + for i := 0; i < int(numSigs); i++ { + if err := ReadElement(r, &sigs[i]); err != nil { + return err + } + } + } + *e = sigs + + case *Sig: + if _, err := io.ReadFull(r, e.bytes[:]); err != nil { + return err + } + + case *ErrorData: + var l [2]byte + if _, err := io.ReadFull(r, l[:]); err != nil { + return err + } + errorLen := binary.BigEndian.Uint16(l[:]) + + *e = ErrorData(make([]byte, errorLen)) + if _, err := io.ReadFull(r, *e); err != nil { + return err + } + + case *[33]byte: + if _, err := io.ReadFull(r, e[:]); err != nil { + return err + } + + case []byte: + if _, err := io.ReadFull(r, e); err != nil { + return err + } + + case *FailCode: + if err := ReadElement(r, (*uint16)(e)); err != nil { + return err + } + + case *ChannelID: + if _, err := io.ReadFull(r, e[:]); err != nil { + return err + } + + case *ShortChannelID: + var blockHeight [4]byte + if _, err = io.ReadFull(r, blockHeight[1:]); err != nil { + return err + } + + var txIndex [4]byte + if _, err = io.ReadFull(r, txIndex[1:]); err != nil { + return err + } + + var txPosition [2]byte + if _, err = io.ReadFull(r, txPosition[:]); err != nil { + return err + } + + *e = ShortChannelID{ + BlockHeight: binary.BigEndian.Uint32(blockHeight[:]), + TxIndex: binary.BigEndian.Uint32(txIndex[:]), + TxPosition: binary.BigEndian.Uint16(txPosition[:]), + } + + case *ExtraOpaqueData: + return e.Decode(r) + + default: + return fmt.Errorf("unknown type in ReadElement: %T", e) + } + + return nil +} + +// ReadElements deserializes a variable number of elements into the passed +// io.Reader, with each element being deserialized according to the ReadElement +// function. +func ReadElements(r io.Reader, elements ...interface{}) error { + for _, element := range elements { + err := ReadElement(r, element) + if err != nil { + return err + } + } + return nil +} diff --git a/payments/db/migration1/lnwire/message.go b/payments/db/migration1/lnwire/message.go new file mode 100644 index 000000000..def762f18 --- /dev/null +++ b/payments/db/migration1/lnwire/message.go @@ -0,0 +1,323 @@ +// Copyright (c) 2013-2017 The btcsuite developers +// Copyright (c) 2015-2016 The Decred developers +// code derived from https://github .com/btcsuite/btcd/blob/master/wire/message.go +// Copyright (C) 2015-2022 The Lightning Network Developers + +package lnwire + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" +) + +// MessageTypeSize is the size in bytes of the message type field in the header +// of all messages. +const MessageTypeSize = 2 + +// MessageType is the unique 2 byte big-endian integer that indicates the type +// of message on the wire. All messages have a very simple header which +// consists simply of 2-byte message type. We omit a length field, and checksum +// as the Lightning Protocol is intended to be encapsulated within a +// confidential+authenticated cryptographic messaging protocol. +type MessageType uint16 + +// The currently defined message types within this current version of the +// Lightning protocol. +const ( + MsgWarning MessageType = 1 + MsgStfu = 2 + MsgInit = 16 + MsgError = 17 + MsgPing = 18 + MsgPong = 19 + MsgOpenChannel = 32 + MsgAcceptChannel = 33 + MsgFundingCreated = 34 + MsgFundingSigned = 35 + MsgChannelReady = 36 + MsgShutdown = 38 + MsgClosingSigned = 39 + MsgClosingComplete = 40 + MsgClosingSig = 41 + MsgDynPropose = 111 + MsgDynAck = 113 + MsgDynReject = 115 + MsgDynCommit = 117 + MsgUpdateAddHTLC = 128 + MsgUpdateFulfillHTLC = 130 + MsgUpdateFailHTLC = 131 + MsgCommitSig = 132 + MsgRevokeAndAck = 133 + MsgUpdateFee = 134 + MsgUpdateFailMalformedHTLC = 135 + MsgChannelReestablish = 136 + MsgChannelAnnouncement = 256 + MsgNodeAnnouncement = 257 + MsgChannelUpdate = 258 + MsgAnnounceSignatures = 259 + MsgAnnounceSignatures2 = 260 + MsgQueryShortChanIDs = 261 + MsgReplyShortChanIDsEnd = 262 + MsgQueryChannelRange = 263 + MsgReplyChannelRange = 264 + MsgGossipTimestampRange = 265 + MsgChannelAnnouncement2 = 267 + MsgNodeAnnouncement2 = 269 + MsgChannelUpdate2 = 271 + MsgOnionMessage = 513 + MsgKickoffSig = 777 + + // MsgEnd defines the end of the official message range of the protocol. + // If a new message is added beyond this message, then this should be + // modified. + MsgEnd = 778 +) + +// IsChannelUpdate is a filter function that discerns channel update messages +// from the other messages in the Lightning Network Protocol. +func (t MessageType) IsChannelUpdate() bool { + switch t { + case MsgUpdateAddHTLC: + return true + case MsgUpdateFulfillHTLC: + return true + case MsgUpdateFailHTLC: + return true + case MsgUpdateFailMalformedHTLC: + return true + case MsgUpdateFee: + return true + default: + return false + } +} + +// ErrorEncodeMessage is used when failed to encode the message payload. +func ErrorEncodeMessage(err error) error { + return fmt.Errorf("failed to encode message to buffer, got %w", err) +} + +// ErrorWriteMessageType is used when failed to write the message type. +func ErrorWriteMessageType(err error) error { + return fmt.Errorf("failed to write message type, got %w", err) +} + +// ErrorPayloadTooLarge is used when the payload size exceeds the +// MaxMsgBody. +func ErrorPayloadTooLarge(size int) error { + return fmt.Errorf( + "message payload is too large - encoded %d bytes, "+ + "but maximum message payload is %d bytes", + size, MaxMsgBody, + ) +} + +// String return the string representation of message type. +func (t MessageType) String() string { + switch t { + case MsgWarning: + return "Warning" + case MsgStfu: + return "Stfu" + case MsgInit: + return "Init" + case MsgOpenChannel: + return "MsgOpenChannel" + case MsgAcceptChannel: + return "MsgAcceptChannel" + case MsgFundingCreated: + return "MsgFundingCreated" + case MsgFundingSigned: + return "MsgFundingSigned" + case MsgChannelReady: + return "ChannelReady" + case MsgShutdown: + return "Shutdown" + case MsgClosingSigned: + return "ClosingSigned" + case MsgDynPropose: + return "DynPropose" + case MsgDynAck: + return "DynAck" + case MsgDynReject: + return "DynReject" + case MsgDynCommit: + return "DynCommit" + case MsgKickoffSig: + return "KickoffSig" + case MsgUpdateAddHTLC: + return "UpdateAddHTLC" + case MsgUpdateFailHTLC: + return "UpdateFailHTLC" + case MsgUpdateFulfillHTLC: + return "UpdateFulfillHTLC" + case MsgCommitSig: + return "CommitSig" + case MsgRevokeAndAck: + return "RevokeAndAck" + case MsgUpdateFailMalformedHTLC: + return "UpdateFailMalformedHTLC" + case MsgChannelReestablish: + return "ChannelReestablish" + case MsgError: + return "Error" + case MsgChannelAnnouncement: + return "ChannelAnnouncement" + case MsgChannelUpdate: + return "ChannelUpdate" + case MsgNodeAnnouncement: + return "NodeAnnouncement1" + case MsgPing: + return "Ping" + case MsgAnnounceSignatures: + return "AnnounceSignatures" + case MsgPong: + return "Pong" + case MsgUpdateFee: + return "UpdateFee" + case MsgQueryShortChanIDs: + return "QueryShortChanIDs" + case MsgReplyShortChanIDsEnd: + return "ReplyShortChanIDsEnd" + case MsgQueryChannelRange: + return "QueryChannelRange" + case MsgReplyChannelRange: + return "ReplyChannelRange" + case MsgGossipTimestampRange: + return "GossipTimestampRange" + case MsgClosingComplete: + return "ClosingComplete" + case MsgClosingSig: + return "ClosingSig" + case MsgAnnounceSignatures2: + return "MsgAnnounceSignatures2" + case MsgChannelAnnouncement2: + return "ChannelAnnouncement2" + case MsgNodeAnnouncement2: + return "NodeAnnouncement2" + case MsgChannelUpdate2: + return "ChannelUpdate2" + case MsgOnionMessage: + return "OnionMessage" + default: + return "" + } +} + +// UnknownMessage is an implementation of the error interface that allows the +// creation of an error in response to an unknown message. +type UnknownMessage struct { + messageType MessageType +} + +// Error returns a human readable string describing the error. +// +// This is part of the error interface. +func (u *UnknownMessage) Error() string { + return fmt.Sprintf("unable to parse message of unknown type: %v", + u.messageType) +} + +// Serializable is an interface which defines a lightning wire serializable +// object. +type Serializable interface { + // Decode reads the bytes stream and converts it to the object. + Decode(io.Reader, uint32) error + + // Encode converts object to the bytes stream and write it into the + // write buffer. + Encode(*bytes.Buffer, uint32) error +} + +// Message is an interface that defines a lightning wire protocol message. The +// interface is general in order to allow implementing types full control over +// the representation of its data. +type Message interface { + Serializable + MsgType() MessageType +} + +// LinkUpdater is an interface implemented by most messages in BOLT 2 that are +// allowed to update the channel state. +type LinkUpdater interface { + // All LinkUpdater messages are messages and so we embed the interface + // so that we can treat it as a message if all we know about it is that + // it is a LinkUpdater message. + Message + + // TargetChanID returns the channel id of the link for which this + // message is intended. + TargetChanID() ChannelID +} + +// SizeableMessage is an interface that extends the base Message interface with +// a method to calculate the serialized size of a message. +type SizeableMessage interface { + Message + + // SerializedSize returns the serialized size of the message in bytes. + // The returned size includes the message type header bytes. + SerializedSize() (uint32, error) +} + +// MessageSerializedSize calculates the serialized size of a message in bytes. +// This is a helper function that can be used by all message types to implement +// the SerializedSize method. +func MessageSerializedSize(msg Message) (uint32, error) { + var buf bytes.Buffer + + // Encode the message to the buffer. + if err := msg.Encode(&buf, 0); err != nil { + return 0, err + } + + // Add the size of the message type. + return uint32(buf.Len()) + MessageTypeSize, nil +} + +// WriteMessage writes a lightning Message to a buffer including the necessary +// header information and returns the number of bytes written. If any error is +// encountered, the buffer passed will be reset to its original state since we +// don't want any broken bytes left. In other words, no bytes will be written +// if there's an error. Either all or none of the message bytes will be written +// to the buffer. +// +// NOTE: this method is not concurrent safe. +func WriteMessage(buf *bytes.Buffer, msg Message, pver uint32) (int, error) { + // Record the size of the bytes already written in buffer. + oldByteSize := buf.Len() + + // cleanBrokenBytes is a helper closure that helps reset the buffer to + // its original state. It truncates all the bytes written in current + // scope. + var cleanBrokenBytes = func(b *bytes.Buffer) int { + b.Truncate(oldByteSize) + return 0 + } + + // Write the message type. + var mType [2]byte + binary.BigEndian.PutUint16(mType[:], uint16(msg.MsgType())) + msgTypeBytes, err := buf.Write(mType[:]) + if err != nil { + return cleanBrokenBytes(buf), ErrorWriteMessageType(err) + } + + // Use the write buffer to encode our message. + if err := msg.Encode(buf, pver); err != nil { + return cleanBrokenBytes(buf), ErrorEncodeMessage(err) + } + + // Enforce maximum overall message payload. The write buffer now has + // the size of len(originalBytes) + len(payload) + len(type). We want + // to enforce the payload here, so we subtract it by the length of the + // type and old bytes. + lenp := buf.Len() - oldByteSize - msgTypeBytes + if lenp > MaxMsgBody { + return cleanBrokenBytes(buf), ErrorPayloadTooLarge(lenp) + } + + return buf.Len() - oldByteSize, nil +} diff --git a/payments/db/migration1/lnwire/msat.go b/payments/db/migration1/lnwire/msat.go new file mode 100644 index 000000000..7d6d581e1 --- /dev/null +++ b/payments/db/migration1/lnwire/msat.go @@ -0,0 +1,92 @@ +package lnwire + +import ( + "fmt" + "io" + + "github.com/btcsuite/btcd/btcutil" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // mSatScale is a value that's used to scale satoshis to milli-satoshis, and + // the other way around. + mSatScale uint64 = 1000 + + // MaxMilliSatoshi is the maximum number of msats that can be expressed + // in this data type. + MaxMilliSatoshi = ^MilliSatoshi(0) +) + +// MilliSatoshi are the native unit of the Lightning Network. A milli-satoshi +// is simply 1/1000th of a satoshi. There are 1000 milli-satoshis in a single +// satoshi. Within the network, all HTLC payments are denominated in +// milli-satoshis. As milli-satoshis aren't deliverable on the native +// blockchain, before settling to broadcasting, the values are rounded down to +// the nearest satoshi. +type MilliSatoshi uint64 + +// NewMSatFromSatoshis creates a new MilliSatoshi instance from a target amount +// of satoshis. +func NewMSatFromSatoshis(sat btcutil.Amount) MilliSatoshi { + return MilliSatoshi(uint64(sat) * mSatScale) +} + +// ToBTC converts the target MilliSatoshi amount to its corresponding value +// when expressed in BTC. +func (m MilliSatoshi) ToBTC() float64 { + sat := m.ToSatoshis() + return sat.ToBTC() +} + +// ToSatoshis converts the target MilliSatoshi amount to satoshis. Simply, this +// sheds a factor of 1000 from the mSAT amount in order to convert it to SAT. +func (m MilliSatoshi) ToSatoshis() btcutil.Amount { + return btcutil.Amount(uint64(m) / mSatScale) +} + +// String returns the string representation of the mSAT amount. +func (m MilliSatoshi) String() string { + return fmt.Sprintf("%v mSAT", uint64(m)) +} + +// TODO(roasbeef): extend with arithmetic operations? + +// Record returns a TLV record that can be used to encode/decode a MilliSatoshi +// to/from a TLV stream. +func (m *MilliSatoshi) Record() tlv.Record { + msat := uint64(*m) + + return tlv.MakeDynamicRecord( + 0, m, tlv.SizeBigSize(&msat), encodeMilliSatoshis, + decodeMilliSatoshis, + ) +} + +func encodeMilliSatoshis(w io.Writer, val interface{}, buf *[8]byte) error { + if v, ok := val.(*MilliSatoshi); ok { + bigSize := uint64(*v) + + return tlv.EBigSize(w, &bigSize, buf) + } + + return tlv.NewTypeForEncodingErr(val, "lnwire.MilliSatoshi") +} + +func decodeMilliSatoshis(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if v, ok := val.(*MilliSatoshi); ok { + var bigSize uint64 + err := tlv.DBigSize(r, &bigSize, buf, l) + if err != nil { + return err + } + + *v = MilliSatoshi(bigSize) + + return nil + } + + return tlv.NewTypeForDecodingErr(val, "lnwire.MilliSatoshi", l, l) +} diff --git a/payments/db/migration1/lnwire/onion_error.go b/payments/db/migration1/lnwire/onion_error.go new file mode 100644 index 000000000..9cc115ff0 --- /dev/null +++ b/payments/db/migration1/lnwire/onion_error.go @@ -0,0 +1,1546 @@ +package lnwire + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/binary" + "fmt" + "io" + + "github.com/davecgh/go-spew/spew" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/tlv" +) + +// FailureMessage represents the onion failure object identified by its unique +// failure code. +type FailureMessage interface { + // Code returns a failure code describing the exact nature of the + // error. + Code() FailCode + + // Error returns a human readable string describing the error. With + // this method, the FailureMessage interface meets the built-in error + // interface. + Error() string +} + +// FailureMessageLength is the size of the failure message plus the size of +// padding. The FailureMessage message should always be EXACTLY this size. +const FailureMessageLength = 256 + +const ( + // FlagBadOnion error flag describes an unparsable, encrypted by + // previous node. + FlagBadOnion FailCode = 0x8000 + + // FlagPerm error flag indicates a permanent failure. + FlagPerm FailCode = 0x4000 + + // FlagNode error flag indicates a node failure. + FlagNode FailCode = 0x2000 + + // FlagUpdate error flag indicates a new channel update is enclosed + // within the error. + FlagUpdate FailCode = 0x1000 +) + +// FailCode specifies the precise reason that an upstream HTLC was canceled. +// Each UpdateFailHTLC message carries a FailCode which is to be passed +// backwards, encrypted at each step back to the source of the HTLC within the +// route. +type FailCode uint16 + +// The currently defined onion failure types within this current version of the +// Lightning protocol. +const ( + CodeNone FailCode = 0 + CodeInvalidRealm = FlagBadOnion | 1 + CodeTemporaryNodeFailure = FlagNode | 2 + CodePermanentNodeFailure = FlagPerm | FlagNode | 2 + CodeRequiredNodeFeatureMissing = FlagPerm | FlagNode | 3 + CodeInvalidOnionVersion = FlagBadOnion | FlagPerm | 4 + CodeInvalidOnionHmac = FlagBadOnion | FlagPerm | 5 + CodeInvalidOnionKey = FlagBadOnion | FlagPerm | 6 + CodeTemporaryChannelFailure = FlagUpdate | 7 + CodePermanentChannelFailure = FlagPerm | 8 + CodeRequiredChannelFeatureMissing = FlagPerm | 9 + CodeUnknownNextPeer = FlagPerm | 10 + CodeAmountBelowMinimum = FlagUpdate | 11 + CodeFeeInsufficient = FlagUpdate | 12 + CodeIncorrectCltvExpiry = FlagUpdate | 13 + CodeExpiryTooSoon = FlagUpdate | 14 + CodeChannelDisabled = FlagUpdate | 20 + CodeIncorrectOrUnknownPaymentDetails = FlagPerm | 15 + CodeIncorrectPaymentAmount = FlagPerm | 16 + CodeFinalExpiryTooSoon FailCode = 17 + CodeFinalIncorrectCltvExpiry FailCode = 18 + CodeFinalIncorrectHtlcAmount FailCode = 19 + CodeExpiryTooFar FailCode = 21 + CodeInvalidOnionPayload = FlagPerm | 22 + CodeMPPTimeout FailCode = 23 + CodeInvalidBlinding = FlagBadOnion | FlagPerm | 24 //nolint:ll +) + +// String returns the string representation of the failure code. +func (c FailCode) String() string { + switch c { + case CodeInvalidRealm: + return "InvalidRealm" + + case CodeTemporaryNodeFailure: + return "TemporaryNodeFailure" + + case CodePermanentNodeFailure: + return "PermanentNodeFailure" + + case CodeRequiredNodeFeatureMissing: + return "RequiredNodeFeatureMissing" + + case CodeInvalidOnionVersion: + return "InvalidOnionVersion" + + case CodeInvalidOnionHmac: + return "InvalidOnionHmac" + + case CodeInvalidOnionKey: + return "InvalidOnionKey" + + case CodeTemporaryChannelFailure: + return "TemporaryChannelFailure" + + case CodePermanentChannelFailure: + return "PermanentChannelFailure" + + case CodeRequiredChannelFeatureMissing: + return "RequiredChannelFeatureMissing" + + case CodeUnknownNextPeer: + return "UnknownNextPeer" + + case CodeAmountBelowMinimum: + return "AmountBelowMinimum" + + case CodeFeeInsufficient: + return "FeeInsufficient" + + case CodeIncorrectCltvExpiry: + return "IncorrectCltvExpiry" + + case CodeIncorrectPaymentAmount: + return "IncorrectPaymentAmount" + + case CodeExpiryTooSoon: + return "ExpiryTooSoon" + + case CodeChannelDisabled: + return "ChannelDisabled" + + case CodeIncorrectOrUnknownPaymentDetails: + return "IncorrectOrUnknownPaymentDetails" + + case CodeFinalExpiryTooSoon: + return "FinalExpiryTooSoon" + + case CodeFinalIncorrectCltvExpiry: + return "FinalIncorrectCltvExpiry" + + case CodeFinalIncorrectHtlcAmount: + return "FinalIncorrectHtlcAmount" + + case CodeExpiryTooFar: + return "ExpiryTooFar" + + case CodeInvalidOnionPayload: + return "InvalidOnionPayload" + + case CodeMPPTimeout: + return "MPPTimeout" + + case CodeInvalidBlinding: + return "InvalidBlinding" + + default: + return "" + } +} + +// FailInvalidRealm is returned if the realm byte is unknown. +// +// NOTE: May be returned by any node in the payment route. +type FailInvalidRealm struct{} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailInvalidRealm) Error() string { + return f.Code().String() +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailInvalidRealm) Code() FailCode { + return CodeInvalidRealm +} + +// FailTemporaryNodeFailure is returned if an otherwise unspecified transient +// error occurs for the entire node. +// +// NOTE: May be returned by any node in the payment route. +type FailTemporaryNodeFailure struct{} + +// Code returns the failure unique code. +// NOTE: Part of the FailureMessage interface. +func (f *FailTemporaryNodeFailure) Code() FailCode { + return CodeTemporaryNodeFailure +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailTemporaryNodeFailure) Error() string { + return f.Code().String() +} + +// FailPermanentNodeFailure is returned if an otherwise unspecified permanent +// error occurs for the entire node. +// +// NOTE: May be returned by any node in the payment route. +type FailPermanentNodeFailure struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailPermanentNodeFailure) Code() FailCode { + return CodePermanentNodeFailure +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailPermanentNodeFailure) Error() string { + return f.Code().String() +} + +// FailRequiredNodeFeatureMissing is returned if a node has requirement +// advertised in its node_announcement features which were not present in the +// onion. +// +// NOTE: May be returned by any node in the payment route. +type FailRequiredNodeFeatureMissing struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailRequiredNodeFeatureMissing) Code() FailCode { + return CodeRequiredNodeFeatureMissing +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailRequiredNodeFeatureMissing) Error() string { + return f.Code().String() +} + +// FailPermanentChannelFailure is return if an otherwise unspecified permanent +// error occurs for the outgoing channel (eg. channel (recently). +// +// NOTE: May be returned by any node in the payment route. +type FailPermanentChannelFailure struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailPermanentChannelFailure) Code() FailCode { + return CodePermanentChannelFailure +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailPermanentChannelFailure) Error() string { + return f.Code().String() +} + +// FailRequiredChannelFeatureMissing is returned if the outgoing channel has a +// requirement advertised in its channel announcement features which were not +// present in the onion. +// +// NOTE: May only be returned by intermediate nodes. +type FailRequiredChannelFeatureMissing struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailRequiredChannelFeatureMissing) Code() FailCode { + return CodeRequiredChannelFeatureMissing +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailRequiredChannelFeatureMissing) Error() string { + return f.Code().String() +} + +// FailUnknownNextPeer is returned if the next peer specified by the onion is +// not known. +// +// NOTE: May only be returned by intermediate nodes. +type FailUnknownNextPeer struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailUnknownNextPeer) Code() FailCode { + return CodeUnknownNextPeer +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailUnknownNextPeer) Error() string { + return f.Code().String() +} + +// FailIncorrectPaymentAmount is returned if the amount paid is less than the +// amount expected, the final node MUST fail the HTLC. If the amount paid is +// more than twice the amount expected, the final node SHOULD fail the HTLC. +// This allows the sender to reduce information leakage by altering the amount, +// without allowing accidental gross overpayment. +// +// NOTE: May only be returned by the final node in the path. +type FailIncorrectPaymentAmount struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailIncorrectPaymentAmount) Code() FailCode { + return CodeIncorrectPaymentAmount +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailIncorrectPaymentAmount) Error() string { + return f.Code().String() +} + +// FailIncorrectDetails is returned for two reasons: +// +// 1) if the payment hash has already been paid, the final node MAY treat the +// payment hash as unknown, or may succeed in accepting the HTLC. If the +// payment hash is unknown, the final node MUST fail the HTLC. +// +// 2) if the amount paid is less than the amount expected, the final node MUST +// fail the HTLC. If the amount paid is more than twice the amount expected, +// the final node SHOULD fail the HTLC. This allows the sender to reduce +// information leakage by altering the amount, without allowing accidental +// gross overpayment. +// +// NOTE: May only be returned by the final node in the path. +type FailIncorrectDetails struct { + // amount is the value of the extended HTLC. + amount MilliSatoshi + + // height is the block height when the htlc was received. + height uint32 + + // extraOpaqueData contains additional failure message tlv data. + extraOpaqueData ExtraOpaqueData +} + +// NewFailIncorrectDetails makes a new instance of the FailIncorrectDetails +// error bound to the specified HTLC amount and acceptance height. +func NewFailIncorrectDetails(amt MilliSatoshi, + height uint32) *FailIncorrectDetails { + + return &FailIncorrectDetails{ + amount: amt, + height: height, + extraOpaqueData: []byte{}, + } +} + +// Amount is the value of the extended HTLC. +func (f *FailIncorrectDetails) Amount() MilliSatoshi { + return f.amount +} + +// Height is the block height when the htlc was received. +func (f *FailIncorrectDetails) Height() uint32 { + return f.height +} + +// ExtraOpaqueData returns additional failure message tlv data. +func (f *FailIncorrectDetails) ExtraOpaqueData() ExtraOpaqueData { + return f.extraOpaqueData +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailIncorrectDetails) Code() FailCode { + return CodeIncorrectOrUnknownPaymentDetails +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailIncorrectDetails) Error() string { + return fmt.Sprintf( + "%v(amt=%v, height=%v)", CodeIncorrectOrUnknownPaymentDetails, + f.amount, f.height, + ) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailIncorrectDetails) Decode(r io.Reader, pver uint32) error { + err := ReadElement(r, &f.amount) + switch { + // This is an optional tack on that was added later in the protocol. As + // a result, older nodes may not include this value. We'll account for + // this by checking for io.EOF here which means that no bytes were read + // at all. + case err == io.EOF: + return nil + + case err != nil: + return err + } + + // At a later stage, the height field was also tacked on. We need to + // check for io.EOF here as well. + err = ReadElement(r, &f.height) + switch { + case err == io.EOF: + return nil + + case err != nil: + return err + } + + return f.extraOpaqueData.Decode(r) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailIncorrectDetails) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteMilliSatoshi(w, f.amount); err != nil { + return err + } + + if err := WriteUint32(w, f.height); err != nil { + return err + } + + return f.extraOpaqueData.Encode(w) +} + +// FailFinalExpiryTooSoon is returned if the cltv_expiry is too low, the final +// node MUST fail the HTLC. +// +// NOTE: May only be returned by the final node in the path. +type FailFinalExpiryTooSoon struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailFinalExpiryTooSoon) Code() FailCode { + return CodeFinalExpiryTooSoon +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailFinalExpiryTooSoon) Error() string { + return f.Code().String() +} + +// NewFinalExpiryTooSoon creates new instance of the FailFinalExpiryTooSoon. +func NewFinalExpiryTooSoon() *FailFinalExpiryTooSoon { + return &FailFinalExpiryTooSoon{} +} + +// FailInvalidOnionVersion is returned if the onion version byte is unknown. +// +// NOTE: May be returned only by intermediate nodes. +type FailInvalidOnionVersion struct { + // OnionSHA256 hash of the onion blob which haven't been proceeded. + OnionSHA256 [sha256.Size]byte +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailInvalidOnionVersion) Error() string { + return fmt.Sprintf("InvalidOnionVersion(onion_sha=%x)", f.OnionSHA256[:]) +} + +// NewInvalidOnionVersion creates new instance of the FailInvalidOnionVersion. +func NewInvalidOnionVersion(onion []byte) *FailInvalidOnionVersion { + return &FailInvalidOnionVersion{OnionSHA256: sha256.Sum256(onion)} +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailInvalidOnionVersion) Code() FailCode { + return CodeInvalidOnionVersion +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionVersion) Decode(r io.Reader, pver uint32) error { + return ReadElement(r, f.OnionSHA256[:]) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionVersion) Encode(w *bytes.Buffer, pver uint32) error { + return WriteBytes(w, f.OnionSHA256[:]) +} + +// FailInvalidOnionHmac is return if the onion HMAC is incorrect. +// +// NOTE: May only be returned by intermediate nodes. +type FailInvalidOnionHmac struct { + // OnionSHA256 hash of the onion blob which haven't been proceeded. + OnionSHA256 [sha256.Size]byte +} + +// NewInvalidOnionHmac creates new instance of the FailInvalidOnionHmac. +func NewInvalidOnionHmac(onion []byte) *FailInvalidOnionHmac { + return &FailInvalidOnionHmac{OnionSHA256: sha256.Sum256(onion)} +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailInvalidOnionHmac) Code() FailCode { + return CodeInvalidOnionHmac +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionHmac) Decode(r io.Reader, pver uint32) error { + return ReadElement(r, f.OnionSHA256[:]) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionHmac) Encode(w *bytes.Buffer, pver uint32) error { + return WriteBytes(w, f.OnionSHA256[:]) +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailInvalidOnionHmac) Error() string { + return fmt.Sprintf("InvalidOnionHMAC(onion_sha=%x)", f.OnionSHA256[:]) +} + +// FailInvalidOnionKey is return if the ephemeral key in the onion is +// unparsable. +// +// NOTE: May only be returned by intermediate nodes. +type FailInvalidOnionKey struct { + // OnionSHA256 hash of the onion blob which haven't been proceeded. + OnionSHA256 [sha256.Size]byte +} + +// NewInvalidOnionKey creates new instance of the FailInvalidOnionKey. +func NewInvalidOnionKey(onion []byte) *FailInvalidOnionKey { + return &FailInvalidOnionKey{OnionSHA256: sha256.Sum256(onion)} +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailInvalidOnionKey) Code() FailCode { + return CodeInvalidOnionKey +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionKey) Decode(r io.Reader, pver uint32) error { + return ReadElement(r, f.OnionSHA256[:]) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionKey) Encode(w *bytes.Buffer, pver uint32) error { + return WriteBytes(w, f.OnionSHA256[:]) +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailInvalidOnionKey) Error() string { + return fmt.Sprintf("InvalidOnionKey(onion_sha=%x)", f.OnionSHA256[:]) +} + +// parseChannelUpdateCompatibilityMode will attempt to parse a channel updated +// encoded into an onion error payload in two ways. First, we'll try the +// compatibility oriented version wherein we'll _skip_ the length prefixing on +// the channel update message. Older versions of c-lighting do this so we'll +// attempt to parse these messages in order to retain compatibility. If we're +// unable to pull out a fully valid version, then we'll fall back to the +// regular parsing mechanism which includes the length prefix an NO type byte. +func parseChannelUpdateCompatibilityMode(reader io.Reader, length uint16, + chanUpdate *ChannelUpdate1, pver uint32) error { + + // Instantiate a LimitReader because there may be additional data + // present after the channel update. Without limiting the stream, the + // additional data would be interpreted as channel update tlv data. + limitReader := io.LimitReader(reader, int64(length)) + + r := bufio.NewReader(limitReader) + + // We'll peek out two bytes from the buffer without advancing the + // buffer so we can decide how to parse the remainder of it. + maybeTypeBytes, err := r.Peek(2) + if err != nil { + return err + } + + // Some nodes well prefix an additional set of bytes in front of their + // channel updates. These bytes will _almost_ always be 258 or the type + // of the ChannelUpdate message. + typeInt := binary.BigEndian.Uint16(maybeTypeBytes) + if typeInt == MsgChannelUpdate { + // At this point it's likely the case that this is a channel + // update message with its type prefixed, so we'll snip off the + // first two bytes and parse it as normal. + var throwAwayTypeBytes [2]byte + _, err := r.Read(throwAwayTypeBytes[:]) + if err != nil { + return err + } + } + + // At this pint, we've either decided to keep the entire thing, or snip + // off the first two bytes. In either case, we can just read it as + // normal. + return chanUpdate.Decode(r, pver) +} + +// FailTemporaryChannelFailure is if an otherwise unspecified transient error +// occurs for the outgoing channel (eg. channel capacity reached, too many +// in-flight htlcs) +// +// NOTE: May only be returned by intermediate nodes. +type FailTemporaryChannelFailure struct { + // Update is used to update information about state of the channel + // which caused the failure. + // + // NOTE: This field is optional. + Update *ChannelUpdate1 +} + +// NewTemporaryChannelFailure creates new instance of the FailTemporaryChannelFailure. +func NewTemporaryChannelFailure( + update *ChannelUpdate1) *FailTemporaryChannelFailure { + + return &FailTemporaryChannelFailure{Update: update} +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailTemporaryChannelFailure) Code() FailCode { + return CodeTemporaryChannelFailure +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailTemporaryChannelFailure) Error() string { + if f.Update == nil { + return f.Code().String() + } + + return fmt.Sprintf("TemporaryChannelFailure(update=%v)", + spew.Sdump(f.Update)) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailTemporaryChannelFailure) Decode(r io.Reader, pver uint32) error { + var length uint16 + err := ReadElement(r, &length) + if err != nil { + return err + } + + if length != 0 { + f.Update = &ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, f.Update, pver, + ) + } + + return nil +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailTemporaryChannelFailure) Encode(w *bytes.Buffer, + pver uint32) error { + + if f.Update != nil { + return writeOnionErrorChanUpdate(w, f.Update, pver) + } + + // Write zero length to indicate no channel_update is present. + return WriteUint16(w, 0) +} + +// FailAmountBelowMinimum is returned if the HTLC does not reach the current +// minimum amount, we tell them the amount of the incoming HTLC and the current +// channel setting for the outgoing channel. +// +// NOTE: May only be returned by the intermediate nodes in the path. +type FailAmountBelowMinimum struct { + // HtlcMsat is the wrong amount of the incoming HTLC. + HtlcMsat MilliSatoshi + + // Update is used to update information about state of the channel + // which caused the failure. + Update ChannelUpdate1 +} + +// NewAmountBelowMinimum creates new instance of the FailAmountBelowMinimum. +func NewAmountBelowMinimum(htlcMsat MilliSatoshi, + update ChannelUpdate1) *FailAmountBelowMinimum { + + return &FailAmountBelowMinimum{ + HtlcMsat: htlcMsat, + Update: update, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailAmountBelowMinimum) Code() FailCode { + return CodeAmountBelowMinimum +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailAmountBelowMinimum) Error() string { + return fmt.Sprintf("AmountBelowMinimum(amt=%v, update=%v", f.HtlcMsat, + spew.Sdump(f.Update)) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailAmountBelowMinimum) Decode(r io.Reader, pver uint32) error { + if err := ReadElement(r, &f.HtlcMsat); err != nil { + return err + } + + var length uint16 + if err := ReadElement(r, &length); err != nil { + return err + } + + f.Update = ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, &f.Update, pver, + ) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailAmountBelowMinimum) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteMilliSatoshi(w, f.HtlcMsat); err != nil { + return err + } + + return writeOnionErrorChanUpdate(w, &f.Update, pver) +} + +// FailFeeInsufficient is returned if the HTLC does not pay sufficient fee, we +// tell them the amount of the incoming HTLC and the current channel setting +// for the outgoing channel. +// +// NOTE: May only be returned by intermediate nodes. +type FailFeeInsufficient struct { + // HtlcMsat is the wrong amount of the incoming HTLC. + HtlcMsat MilliSatoshi + + // Update is used to update information about state of the channel + // which caused the failure. + Update ChannelUpdate1 +} + +// NewFeeInsufficient creates new instance of the FailFeeInsufficient. +func NewFeeInsufficient(htlcMsat MilliSatoshi, + update ChannelUpdate1) *FailFeeInsufficient { + return &FailFeeInsufficient{ + HtlcMsat: htlcMsat, + Update: update, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailFeeInsufficient) Code() FailCode { + return CodeFeeInsufficient +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailFeeInsufficient) Error() string { + return fmt.Sprintf("FeeInsufficient(htlc_amt==%v, update=%v", f.HtlcMsat, + spew.Sdump(f.Update)) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFeeInsufficient) Decode(r io.Reader, pver uint32) error { + if err := ReadElement(r, &f.HtlcMsat); err != nil { + return err + } + + var length uint16 + if err := ReadElement(r, &length); err != nil { + return err + } + + f.Update = ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, &f.Update, pver, + ) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFeeInsufficient) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteMilliSatoshi(w, f.HtlcMsat); err != nil { + return err + } + + return writeOnionErrorChanUpdate(w, &f.Update, pver) +} + +// FailIncorrectCltvExpiry is returned if outgoing cltv value does not match +// the update add htlc's cltv expiry minus cltv expiry delta for the outgoing +// channel, we tell them the cltv expiry and the current channel setting for +// the outgoing channel. +// +// NOTE: May only be returned by intermediate nodes. +type FailIncorrectCltvExpiry struct { + // CltvExpiry is the wrong absolute timeout in blocks, after which + // outgoing HTLC expires. + CltvExpiry uint32 + + // Update is used to update information about state of the channel + // which caused the failure. + Update ChannelUpdate1 +} + +// NewIncorrectCltvExpiry creates new instance of the FailIncorrectCltvExpiry. +func NewIncorrectCltvExpiry(cltvExpiry uint32, + update ChannelUpdate1) *FailIncorrectCltvExpiry { + + return &FailIncorrectCltvExpiry{ + CltvExpiry: cltvExpiry, + Update: update, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailIncorrectCltvExpiry) Code() FailCode { + return CodeIncorrectCltvExpiry +} + +func (f *FailIncorrectCltvExpiry) Error() string { + return fmt.Sprintf("IncorrectCltvExpiry(expiry=%v, update=%v", + f.CltvExpiry, spew.Sdump(f.Update)) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailIncorrectCltvExpiry) Decode(r io.Reader, pver uint32) error { + if err := ReadElement(r, &f.CltvExpiry); err != nil { + return err + } + + var length uint16 + if err := ReadElement(r, &length); err != nil { + return err + } + + f.Update = ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, &f.Update, pver, + ) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailIncorrectCltvExpiry) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteUint32(w, f.CltvExpiry); err != nil { + return err + } + + return writeOnionErrorChanUpdate(w, &f.Update, pver) +} + +// FailExpiryTooSoon is returned if the ctlv-expiry is too near, we tell them +// the current channel setting for the outgoing channel. +// +// NOTE: May only be returned by intermediate nodes. +type FailExpiryTooSoon struct { + // Update is used to update information about state of the channel + // which caused the failure. + Update ChannelUpdate1 +} + +// NewExpiryTooSoon creates new instance of the FailExpiryTooSoon. +func NewExpiryTooSoon(update ChannelUpdate1) *FailExpiryTooSoon { + return &FailExpiryTooSoon{ + Update: update, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailExpiryTooSoon) Code() FailCode { + return CodeExpiryTooSoon +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailExpiryTooSoon) Error() string { + return fmt.Sprintf("ExpiryTooSoon(update=%v", spew.Sdump(f.Update)) +} + +// Decode decodes the failure from l stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailExpiryTooSoon) Decode(r io.Reader, pver uint32) error { + var length uint16 + if err := ReadElement(r, &length); err != nil { + return err + } + + f.Update = ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, &f.Update, pver, + ) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailExpiryTooSoon) Encode(w *bytes.Buffer, pver uint32) error { + return writeOnionErrorChanUpdate(w, &f.Update, pver) +} + +// FailChannelDisabled is returned if the channel is disabled, we tell them the +// current channel setting for the outgoing channel. +// +// NOTE: May only be returned by intermediate nodes. +type FailChannelDisabled struct { + // Flags least-significant bit must be set to 0 if the creating node + // corresponds to the first node in the previously sent channel + // announcement and 1 otherwise. + Flags uint16 + + // Update is used to update information about state of the channel + // which caused the failure. + Update ChannelUpdate1 +} + +// NewChannelDisabled creates new instance of the FailChannelDisabled. +func NewChannelDisabled(flags uint16, + update ChannelUpdate1) *FailChannelDisabled { + + return &FailChannelDisabled{ + Flags: flags, + Update: update, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailChannelDisabled) Code() FailCode { + return CodeChannelDisabled +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailChannelDisabled) Error() string { + return fmt.Sprintf("ChannelDisabled(flags=%v, update=%v", f.Flags, + spew.Sdump(f.Update)) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailChannelDisabled) Decode(r io.Reader, pver uint32) error { + if err := ReadElement(r, &f.Flags); err != nil { + return err + } + + var length uint16 + if err := ReadElement(r, &length); err != nil { + return err + } + + f.Update = ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, &f.Update, pver, + ) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailChannelDisabled) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteUint16(w, f.Flags); err != nil { + return err + } + + return writeOnionErrorChanUpdate(w, &f.Update, pver) +} + +// FailFinalIncorrectCltvExpiry is returned if the outgoing_cltv_value does not +// match the ctlv_expiry of the HTLC at the final hop. +// +// NOTE: might be returned by final node only. +type FailFinalIncorrectCltvExpiry struct { + // CltvExpiry is the wrong absolute timeout in blocks, after which + // outgoing HTLC expires. + CltvExpiry uint32 +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailFinalIncorrectCltvExpiry) Error() string { + return fmt.Sprintf("FinalIncorrectCltvExpiry(expiry=%v)", f.CltvExpiry) +} + +// NewFinalIncorrectCltvExpiry creates new instance of the +// FailFinalIncorrectCltvExpiry. +func NewFinalIncorrectCltvExpiry(cltvExpiry uint32) *FailFinalIncorrectCltvExpiry { + return &FailFinalIncorrectCltvExpiry{ + CltvExpiry: cltvExpiry, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailFinalIncorrectCltvExpiry) Code() FailCode { + return CodeFinalIncorrectCltvExpiry +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFinalIncorrectCltvExpiry) Decode(r io.Reader, pver uint32) error { + return ReadElement(r, &f.CltvExpiry) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFinalIncorrectCltvExpiry) Encode(w *bytes.Buffer, + pver uint32) error { + + return WriteUint32(w, f.CltvExpiry) +} + +// FailFinalIncorrectHtlcAmount is returned if the amt_to_forward is higher +// than incoming_htlc_amt of the HTLC at the final hop. +// +// NOTE: May only be returned by the final node. +type FailFinalIncorrectHtlcAmount struct { + // IncomingHTLCAmount is the wrong forwarded htlc amount. + IncomingHTLCAmount MilliSatoshi +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailFinalIncorrectHtlcAmount) Error() string { + return fmt.Sprintf("FinalIncorrectHtlcAmount(amt=%v)", + f.IncomingHTLCAmount) +} + +// NewFinalIncorrectHtlcAmount creates new instance of the +// FailFinalIncorrectHtlcAmount. +func NewFinalIncorrectHtlcAmount(amount MilliSatoshi) *FailFinalIncorrectHtlcAmount { + return &FailFinalIncorrectHtlcAmount{ + IncomingHTLCAmount: amount, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailFinalIncorrectHtlcAmount) Code() FailCode { + return CodeFinalIncorrectHtlcAmount +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFinalIncorrectHtlcAmount) Decode(r io.Reader, pver uint32) error { + return ReadElement(r, &f.IncomingHTLCAmount) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFinalIncorrectHtlcAmount) Encode(w *bytes.Buffer, + pver uint32) error { + + return WriteMilliSatoshi(w, f.IncomingHTLCAmount) +} + +// FailExpiryTooFar is returned if the CLTV expiry in the HTLC is too far in the +// future. +// +// NOTE: May be returned by any node in the payment route. +type FailExpiryTooFar struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailExpiryTooFar) Code() FailCode { + return CodeExpiryTooFar +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailExpiryTooFar) Error() string { + return f.Code().String() +} + +// InvalidOnionPayload is returned if the hop could not process the TLV payload +// enclosed in the onion. +type InvalidOnionPayload struct { + // Type is the TLV type that caused the specific failure. + Type uint64 + + // Offset is the byte offset within the payload where the failure + // occurred. + Offset uint16 +} + +// NewInvalidOnionPayload initializes a new InvalidOnionPayload failure. +func NewInvalidOnionPayload(typ uint64, offset uint16) *InvalidOnionPayload { + return &InvalidOnionPayload{ + Type: typ, + Offset: offset, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *InvalidOnionPayload) Code() FailCode { + return CodeInvalidOnionPayload +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *InvalidOnionPayload) Error() string { + return fmt.Sprintf("%v(type=%v, offset=%d)", + f.Code(), f.Type, f.Offset) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *InvalidOnionPayload) Decode(r io.Reader, pver uint32) error { + var buf [8]byte + typ, err := tlv.ReadVarInt(r, &buf) + if err != nil { + return err + } + f.Type = typ + + return ReadElements(r, &f.Offset) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *InvalidOnionPayload) Encode(w *bytes.Buffer, pver uint32) error { + var buf [8]byte + if err := tlv.WriteVarInt(w, f.Type, &buf); err != nil { + return err + } + + return WriteUint16(w, f.Offset) +} + +// FailMPPTimeout is returned if the complete amount for a multi part payment +// was not received within a reasonable time. +// +// NOTE: May only be returned by the final node in the path. +type FailMPPTimeout struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailMPPTimeout) Code() FailCode { + return CodeMPPTimeout +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailMPPTimeout) Error() string { + return f.Code().String() +} + +// FailInvalidBlinding is returned if there has been a route blinding related +// error. +type FailInvalidBlinding struct { + OnionSHA256 [sha256.Size]byte +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailInvalidBlinding) Code() FailCode { + return CodeInvalidBlinding +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailInvalidBlinding) Error() string { + return f.Code().String() +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidBlinding) Decode(r io.Reader, _ uint32) error { + return ReadElement(r, f.OnionSHA256[:]) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidBlinding) Encode(w *bytes.Buffer, _ uint32) error { + return WriteBytes(w, f.OnionSHA256[:]) +} + +// NewInvalidBlinding creates new instance of FailInvalidBlinding. +func NewInvalidBlinding( + onion fn.Option[[OnionPacketSize]byte]) *FailInvalidBlinding { + // The spec allows empty onion hashes for invalid blinding, so we only + // include our onion hash if it's provided. + if onion.IsNone() { + return &FailInvalidBlinding{} + } + + shaSum := fn.MapOptionZ(onion, func(o [OnionPacketSize]byte) [32]byte { + return sha256.Sum256(o[:]) + }) + + return &FailInvalidBlinding{OnionSHA256: shaSum} +} + +// DecodeFailure decodes, validates, and parses the lnwire onion failure, for +// the provided protocol version. +func DecodeFailure(r io.Reader, pver uint32) (FailureMessage, error) { + // First, we'll parse out the encapsulated failure message itself. This + // is a 2 byte length followed by the payload itself. + var failureLength uint16 + if err := ReadElement(r, &failureLength); err != nil { + return nil, fmt.Errorf("unable to read failure len: %w", err) + } + + failureData := make([]byte, failureLength) + if _, err := io.ReadFull(r, failureData); err != nil { + return nil, fmt.Errorf("unable to full read payload of "+ + "%v: %w", failureLength, err) + } + + // Read the padding. + var padLength uint16 + if err := ReadElement(r, &padLength); err != nil { + return nil, fmt.Errorf("unable to read pad len: %w", err) + } + + if _, err := io.CopyN(io.Discard, r, int64(padLength)); err != nil { + return nil, fmt.Errorf("unable to read padding %w", err) + } + + // Verify that we are at the end of the stream now. + scratch := make([]byte, 1) + _, err := r.Read(scratch) + if err != io.EOF { + return nil, fmt.Errorf("unexpected failure bytes") + } + + // Check the total length. Convert to 32 bits to prevent overflow. + totalLength := uint32(padLength) + uint32(failureLength) + if totalLength < FailureMessageLength { + return nil, fmt.Errorf("failure message too short: "+ + "msg=%v, pad=%v, total=%v", + failureLength, padLength, totalLength) + } + + // Decode the failure message. + dataReader := bytes.NewReader(failureData) + + return DecodeFailureMessage(dataReader, pver) +} + +// DecodeFailureMessage decodes just the failure message, ignoring any padding +// that may be present at the end. +func DecodeFailureMessage(r io.Reader, pver uint32) (FailureMessage, error) { + // Once we have the failure data, we can obtain the failure code from + // the first two bytes of the buffer. + var codeBytes [2]byte + if _, err := io.ReadFull(r, codeBytes[:]); err != nil { + return nil, fmt.Errorf("unable to read failure code: %w", err) + } + failCode := FailCode(binary.BigEndian.Uint16(codeBytes[:])) + + // Create the empty failure by given code and populate the failure with + // additional data if needed. + failure, err := makeEmptyOnionError(failCode) + if err != nil { + return nil, fmt.Errorf("unable to make empty error: %w", err) + } + + // Finally, if this failure has a payload, then we'll read that now as + // well. + switch f := failure.(type) { + case Serializable: + if err := f.Decode(r, pver); err != nil { + return nil, fmt.Errorf("unable to decode error "+ + "update (type=%T): %w", failure, err) + } + } + + return failure, nil +} + +// EncodeFailure encodes, including the necessary onion failure header +// information. +func EncodeFailure(w *bytes.Buffer, failure FailureMessage, pver uint32) error { + var failureMessageBuffer bytes.Buffer + + err := EncodeFailureMessage(&failureMessageBuffer, failure, pver) + if err != nil { + return err + } + + // The combined size of this message must be below the max allowed + // failure message length. + failureMessage := failureMessageBuffer.Bytes() + if len(failureMessage) > FailureMessageLength { + return fmt.Errorf("failure message exceed max "+ + "available size: %v", len(failureMessage)) + } + + // Finally, we'll add some padding in order to ensure that all failure + // messages are fixed size. + pad := make([]byte, FailureMessageLength-len(failureMessage)) + + if err := WriteUint16(w, uint16(len(failureMessage))); err != nil { + return err + } + + if err := WriteBytes(w, failureMessage); err != nil { + return err + } + if err := WriteUint16(w, uint16(len(pad))); err != nil { + return err + } + + return WriteBytes(w, pad) +} + +// EncodeFailureMessage encodes just the failure message without adding a length +// and padding the message for the onion protocol. +func EncodeFailureMessage(w *bytes.Buffer, + failure FailureMessage, pver uint32) error { + + // First, we'll write out the error code itself into the failure + // buffer. + var codeBytes [2]byte + code := uint16(failure.Code()) + binary.BigEndian.PutUint16(codeBytes[:], code) + _, err := w.Write(codeBytes[:]) + if err != nil { + return err + } + + // Next, some message have an additional message payload, if this is + // one of those types, then we'll also encode the error payload as + // well. + switch failure := failure.(type) { + case Serializable: + if err := failure.Encode(w, pver); err != nil { + return err + } + } + + return nil +} + +// makeEmptyOnionError creates a new empty onion error of the proper concrete +// type based on the passed failure code. +func makeEmptyOnionError(code FailCode) (FailureMessage, error) { + switch code { + case CodeInvalidRealm: + return &FailInvalidRealm{}, nil + + case CodeTemporaryNodeFailure: + return &FailTemporaryNodeFailure{}, nil + + case CodePermanentNodeFailure: + return &FailPermanentNodeFailure{}, nil + + case CodeRequiredNodeFeatureMissing: + return &FailRequiredNodeFeatureMissing{}, nil + + case CodePermanentChannelFailure: + return &FailPermanentChannelFailure{}, nil + + case CodeRequiredChannelFeatureMissing: + return &FailRequiredChannelFeatureMissing{}, nil + + case CodeUnknownNextPeer: + return &FailUnknownNextPeer{}, nil + + case CodeIncorrectOrUnknownPaymentDetails: + return &FailIncorrectDetails{}, nil + + case CodeIncorrectPaymentAmount: + return &FailIncorrectPaymentAmount{}, nil + + case CodeFinalExpiryTooSoon: + return &FailFinalExpiryTooSoon{}, nil + + case CodeInvalidOnionVersion: + return &FailInvalidOnionVersion{}, nil + + case CodeInvalidOnionHmac: + return &FailInvalidOnionHmac{}, nil + + case CodeInvalidOnionKey: + return &FailInvalidOnionKey{}, nil + + case CodeTemporaryChannelFailure: + return &FailTemporaryChannelFailure{}, nil + + case CodeAmountBelowMinimum: + return &FailAmountBelowMinimum{}, nil + + case CodeFeeInsufficient: + return &FailFeeInsufficient{}, nil + + case CodeIncorrectCltvExpiry: + return &FailIncorrectCltvExpiry{}, nil + + case CodeExpiryTooSoon: + return &FailExpiryTooSoon{}, nil + + case CodeChannelDisabled: + return &FailChannelDisabled{}, nil + + case CodeFinalIncorrectCltvExpiry: + return &FailFinalIncorrectCltvExpiry{}, nil + + case CodeFinalIncorrectHtlcAmount: + return &FailFinalIncorrectHtlcAmount{}, nil + + case CodeExpiryTooFar: + return &FailExpiryTooFar{}, nil + + case CodeInvalidOnionPayload: + return &InvalidOnionPayload{}, nil + + case CodeMPPTimeout: + return &FailMPPTimeout{}, nil + + case CodeInvalidBlinding: + return &FailInvalidBlinding{}, nil + + default: + return nil, fmt.Errorf("unknown error code: %v", code) + } +} + +// writeOnionErrorChanUpdate writes out a ChannelUpdate using the onion error +// format. The format is that we first write out the true serialized length of +// the channel update, followed by the serialized channel update itself. +func writeOnionErrorChanUpdate(w *bytes.Buffer, chanUpdate *ChannelUpdate1, + pver uint32) error { + + // First, we encode the channel update in a temporary buffer in order + // to get the exact serialized size. + var b bytes.Buffer + updateLen, err := WriteMessage(&b, chanUpdate, pver) + if err != nil { + return err + } + + // Now that we know the size, we can write the length out in the main + // writer. + if err := WriteUint16(w, uint16(updateLen)); err != nil { + return err + } + + // With the length written, we'll then write out the serialized channel + // update. + if _, err := w.Write(b.Bytes()); err != nil { + return err + } + + return nil +} diff --git a/payments/db/migration1/lnwire/short_channel_id.go b/payments/db/migration1/lnwire/short_channel_id.go new file mode 100644 index 000000000..e26575001 --- /dev/null +++ b/payments/db/migration1/lnwire/short_channel_id.go @@ -0,0 +1,105 @@ +package lnwire + +import ( + "fmt" + "io" + + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // AliasScidRecordType is the type of the experimental record to denote + // the alias being used in an option_scid_alias channel. + AliasScidRecordType tlv.Type = 1 +) + +// ShortChannelID represents the set of data which is needed to retrieve all +// necessary data to validate the channel existence. +type ShortChannelID struct { + // BlockHeight is the height of the block where funding transaction + // located. + // + // NOTE: This field is limited to 3 bytes. + BlockHeight uint32 + + // TxIndex is a position of funding transaction within a block. + // + // NOTE: This field is limited to 3 bytes. + TxIndex uint32 + + // TxPosition indicating transaction output which pays to the channel. + TxPosition uint16 +} + +// NewShortChanIDFromInt returns a new ShortChannelID which is the decoded +// version of the compact channel ID encoded within the uint64. The format of +// the compact channel ID is as follows: 3 bytes for the block height, 3 bytes +// for the transaction index, and 2 bytes for the output index. +func NewShortChanIDFromInt(chanID uint64) ShortChannelID { + return ShortChannelID{ + BlockHeight: uint32(chanID >> 40), + TxIndex: uint32(chanID>>16) & 0xFFFFFF, + TxPosition: uint16(chanID), + } +} + +// ToUint64 converts the ShortChannelID into a compact format encoded within a +// uint64 (8 bytes). +func (c ShortChannelID) ToUint64() uint64 { + // TODO(roasbeef): explicit error on overflow? + return ((uint64(c.BlockHeight) << 40) | (uint64(c.TxIndex) << 16) | + (uint64(c.TxPosition))) +} + +// String generates a human-readable representation of the channel ID. +func (c ShortChannelID) String() string { + return fmt.Sprintf("%d:%d:%d", c.BlockHeight, c.TxIndex, c.TxPosition) +} + +// AltString generates a human-readable representation of the channel ID +// with 'x' as a separator. +func (c ShortChannelID) AltString() string { + return fmt.Sprintf("%dx%dx%d", c.BlockHeight, c.TxIndex, c.TxPosition) +} + +// Record returns a TLV record that can be used to encode/decode a +// ShortChannelID to/from a TLV stream. +func (c *ShortChannelID) Record() tlv.Record { + return tlv.MakeStaticRecord( + AliasScidRecordType, c, 8, EShortChannelID, DShortChannelID, + ) +} + +// IsDefault returns true if the ShortChannelID represents the zero value for +// its type. +func (c ShortChannelID) IsDefault() bool { + return c == ShortChannelID{} +} + +// EShortChannelID is an encoder for ShortChannelID. It is exported so other +// packages can use the encoding scheme. +func EShortChannelID(w io.Writer, val interface{}, buf *[8]byte) error { + if v, ok := val.(*ShortChannelID); ok { + return tlv.EUint64T(w, v.ToUint64(), buf) + } + return tlv.NewTypeForEncodingErr(val, "lnwire.ShortChannelID") +} + +// DShortChannelID is a decoder for ShortChannelID. It is exported so other +// packages can use the decoding scheme. +func DShortChannelID(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if v, ok := val.(*ShortChannelID); ok { + var scid uint64 + // tlv.DUint64 forces the length to be 8 bytes. + err := tlv.DUint64(r, &scid, buf, l) + if err != nil { + return err + } + + *v = NewShortChanIDFromInt(scid) + return nil + } + return tlv.NewTypeForDecodingErr(val, "lnwire.ShortChannelID", l, 8) +} diff --git a/payments/db/migration1/lnwire/signature.go b/payments/db/migration1/lnwire/signature.go new file mode 100644 index 000000000..35b039ed0 --- /dev/null +++ b/payments/db/migration1/lnwire/signature.go @@ -0,0 +1,292 @@ +package lnwire + +import ( + "errors" + "fmt" + + "github.com/btcsuite/btcd/btcec/v2/ecdsa" + "github.com/btcsuite/btcd/btcec/v2/schnorr" + "github.com/lightningnetwork/lnd/input" + "github.com/lightningnetwork/lnd/tlv" +) + +var ( + errSigTooShort = errors.New("malformed signature: too short") + errBadLength = errors.New("malformed signature: bad length") + errBadRLength = errors.New("malformed signature: bogus R length") + errBadSLength = errors.New("malformed signature: bogus S length") + errRTooLong = errors.New("R is over 32 bytes long without padding") + errSTooLong = errors.New("S is over 32 bytes long without padding") +) + +// sigType represents the type of signature that is carried within the Sig. +// Today this can either be an ECDSA sig or a schnorr sig. Both of these can +// fit cleanly into 64 bytes. +type sigType uint + +const ( + // sigTypeECDSA represents an ECDSA signature. + sigTypeECDSA sigType = iota + + // sigTypeSchnorr represents a schnorr signature. + sigTypeSchnorr +) + +// Sig is a fixed-sized ECDSA signature or 64-byte schnorr signature. For the +// ECDSA sig, unlike Bitcoin, we use fixed sized signatures on the wire, +// instead of DER encoded signatures. This type provides several methods to +// convert to/from a regular Bitcoin DER encoded signature (raw bytes and +// *ecdsa.Signature). +type Sig struct { + bytes [64]byte + + sigType sigType +} + +// ForceSchnorr forces the signature to be interpreted as a schnorr signature. +// This is useful when reading an HTLC sig off the wire for a taproot channel. +// In this case, in order to obtain an input.Signature, we need to know that +// the sig is a schnorr sig. +func (s *Sig) ForceSchnorr() { + s.sigType = sigTypeSchnorr +} + +// RawBytes returns the raw bytes of signature. +func (s *Sig) RawBytes() []byte { + return s.bytes[:] +} + +// Copy copies the signature into a new Sig instance. +func (s *Sig) Copy() Sig { + var sCopy Sig + copy(sCopy.bytes[:], s.bytes[:]) + sCopy.sigType = s.sigType + + return sCopy +} + +// Record returns a Record that can be used to encode or decode the backing +// object. +// +// This returns a record that serializes the sig as a 64-byte fixed size +// signature. +func (s *Sig) Record() tlv.Record { + // We set a type here as zero as it isn't needed when used as a + // RecordT. + return tlv.MakePrimitiveRecord(0, &s.bytes) +} + +// NewSigFromWireECDSA returns a Sig instance based on an ECDSA signature +// that's already in the 64-byte format we expect. +func NewSigFromWireECDSA(sig []byte) (Sig, error) { + if len(sig) != 64 { + return Sig{}, fmt.Errorf("%w: %v bytes", errSigTooShort, + len(sig)) + } + + var s Sig + copy(s.bytes[:], sig) + + return s, nil +} + +// NewSigFromECDSARawSignature returns a Sig from a Bitcoin raw signature +// encoded in the canonical DER encoding. +func NewSigFromECDSARawSignature(sig []byte) (Sig, error) { + var b [64]byte + + // Check the total length is above the minimal. + if len(sig) < ecdsa.MinSigLen { + return Sig{}, errSigTooShort + } + + // The DER representation is laid out as: + // 0x30 0x02 r 0x02 s + // which means the length of R is the 4th byte and the length of S is + // the second byte after R ends. 0x02 signifies a length-prefixed, + // zero-padded, big-endian bigint. 0x30 signifies a DER signature. + // See the Serialize() method for ecdsa.Signature for details. + + // Reading , remaining: [0x02 r 0x02 s] + sigLen := int(sig[1]) + + // siglen should be less than the entire message and greater than + // the minimal message size. + if sigLen+2 > len(sig) || sigLen+2 < ecdsa.MinSigLen { + return Sig{}, errBadLength + } + + // Reading , remaining: [r 0x02 s] + rLen := int(sig[3]) + + // rLen must be positive and must be able to fit in other elements. + // Assuming s is one byte, then we have 0x30, , 0x20, + // , 0x20, , s, a total of 7 bytes. + if rLen <= 0 || rLen+7 > len(sig) { + return Sig{}, errBadRLength + } + + // Reading , remaining: [s] + sLen := int(sig[5+rLen]) + + // S should be the rest of the string. + // sLen must be positive and must be able to fit in other elements. + // We know r is rLen bytes, and we have 0x30, , 0x20, + // , 0x20, , a total of rLen+6 bytes. + if sLen <= 0 || sLen+rLen+6 > len(sig) { + return Sig{}, errBadSLength + } + + // Check to make sure R and S can both fit into their intended buffers. + // We check S first because these code blocks decrement sLen and rLen + // in the case of a 33-byte 0-padded integer returned from Serialize() + // and rLen is used in calculating array indices for S. We can track + // this with additional variables, but it's more efficient to just + // check S first. + if sLen > 32 { + if (sLen > 33) || (sig[6+rLen] != 0x00) { + return Sig{}, errSTooLong + } + sLen-- + copy(b[64-sLen:], sig[7+rLen:]) + } else { + copy(b[64-sLen:], sig[6+rLen:]) + } + + // Do the same for R as we did for S + if rLen > 32 { + if (rLen > 33) || (sig[4] != 0x00) { + return Sig{}, errRTooLong + } + rLen-- + copy(b[32-rLen:], sig[5:5+rLen]) + } else { + copy(b[32-rLen:], sig[4:4+rLen]) + } + + return Sig{ + bytes: b, + sigType: sigTypeECDSA, + }, nil +} + +// NewSigFromSchnorrRawSignature converts a raw schnorr signature into an +// lnwire.Sig. +func NewSigFromSchnorrRawSignature(sig []byte) (Sig, error) { + var s Sig + copy(s.bytes[:], sig) + s.sigType = sigTypeSchnorr + + return s, nil +} + +// NewSigFromSignature creates a new signature as used on the wire, from an +// existing ecdsa.Signature or schnorr.Signature. +func NewSigFromSignature(e input.Signature) (Sig, error) { + if e == nil { + return Sig{}, fmt.Errorf("cannot decode empty signature") + } + + // Nil is still a valid interface, apparently. So we need a more + // explicit check here. + if ecsig, ok := e.(*ecdsa.Signature); ok && ecsig == nil { + return Sig{}, fmt.Errorf("cannot decode empty signature") + } + + switch ecSig := e.(type) { + // If this is a schnorr signature, then we can just pack it as normal, + // since the default encoding is already 64 bytes. + case *schnorr.Signature: + return NewSigFromSchnorrRawSignature(e.Serialize()) + + // For ECDSA signatures, we'll need to do a bit more work to map the + // signature into a compact 64 byte form. + case *ecdsa.Signature: + // Serialize the signature with all the checks that entails. + return NewSigFromECDSARawSignature(e.Serialize()) + + default: + return Sig{}, fmt.Errorf("unknown wire sig type: %T", ecSig) + } +} + +// ToSignature converts the fixed-sized signature to a input.Signature which +// can be used for signature validation checks. +func (s *Sig) ToSignature() (input.Signature, error) { + switch s.sigType { + case sigTypeSchnorr: + return schnorr.ParseSignature(s.bytes[:]) + + case sigTypeECDSA: + // Parse the signature with strict checks. + sigBytes := s.ToSignatureBytes() + sig, err := ecdsa.ParseDERSignature(sigBytes) + if err != nil { + return nil, err + } + + return sig, nil + + default: + return nil, fmt.Errorf("unknown sig type: %v", s.sigType) + } +} + +// ToSignatureBytes serializes the target fixed-sized signature into the +// encoding of the primary domain for the signature. For ECDSA signatures, this +// is the raw bytes of a DER encoding. +func (s *Sig) ToSignatureBytes() []byte { + switch s.sigType { + // For ECDSA signatures, we'll convert to DER encoding. + case sigTypeECDSA: + // Extract canonically-padded bigint representations from buffer + r := extractCanonicalPadding(s.bytes[0:32]) + s := extractCanonicalPadding(s.bytes[32:64]) + rLen := uint8(len(r)) + sLen := uint8(len(s)) + + // Create a canonical serialized signature. DER format is: + // 0x30 0x02 r 0x02 s + sigBytes := make([]byte, 6+rLen+sLen) + sigBytes[0] = 0x30 // DER signature magic value + sigBytes[1] = 4 + rLen + sLen // Length of rest of signature + sigBytes[2] = 0x02 // Big integer magic value + sigBytes[3] = rLen // Length of R + sigBytes[rLen+4] = 0x02 // Big integer magic value + sigBytes[rLen+5] = sLen // Length of S + copy(sigBytes[4:], r) // Copy R + copy(sigBytes[rLen+6:], s) // Copy S + + return sigBytes + + // For schnorr signatures, we can use the same internal 64 bytes. + case sigTypeSchnorr: + // We'll make a copy of the signature so we don't return a + // reference into the raw slice. + var sig [64]byte + copy(sig[:], s.bytes[:]) + return sig[:] + + default: + // TODO(roasbeef): can only be called via public methods so + // never reachable? + panic("sig type not set") + } +} + +// extractCanonicalPadding is a utility function to extract the canonical +// padding of a big-endian integer from the wire encoding (a 0-padded +// big-endian integer) such that it passes btcec.canonicalPadding test. +func extractCanonicalPadding(b []byte) []byte { + for i := 0; i < len(b); i++ { + // Found first non-zero byte. + if b[i] > 0 { + // If the MSB is set, we need zero padding. + if b[i]&0x80 == 0x80 { + return append([]byte{0x00}, b[i:]...) + } + return b[i:] + } + } + return []byte{0x00} +} diff --git a/payments/db/migration1/lnwire/typed_fee.go b/payments/db/migration1/lnwire/typed_fee.go new file mode 100644 index 000000000..f9b6c8d01 --- /dev/null +++ b/payments/db/migration1/lnwire/typed_fee.go @@ -0,0 +1,60 @@ +package lnwire + +import ( + "io" + + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + FeeRecordType tlv.Type = 55555 +) + +// Fee represents a fee schedule. +type Fee struct { + BaseFee int32 + FeeRate int32 +} + +// Record returns a TLV record that can be used to encode/decode the fee +// type from a given TLV stream. +func (l *Fee) Record() tlv.Record { + return tlv.MakeStaticRecord( + FeeRecordType, l, 8, feeEncoder, feeDecoder, + ) +} + +// feeEncoder is a custom TLV encoder for the fee record. +func feeEncoder(w io.Writer, val interface{}, buf *[8]byte) error { + v, ok := val.(*Fee) + if !ok { + return tlv.NewTypeForEncodingErr(val, "lnwire.Fee") + } + + if err := tlv.EUint32T(w, uint32(v.BaseFee), buf); err != nil { + return err + } + + return tlv.EUint32T(w, uint32(v.FeeRate), buf) +} + +// feeDecoder is a custom TLV decoder for the fee record. +func feeDecoder(r io.Reader, val interface{}, buf *[8]byte, l uint64) error { + v, ok := val.(*Fee) + if !ok || l != 8 { + return tlv.NewTypeForDecodingErr(val, "lnwire.Fee", l, 8) + } + + var baseFee, feeRate uint32 + if err := tlv.DUint32(r, &baseFee, buf, 4); err != nil { + return err + } + if err := tlv.DUint32(r, &feeRate, buf, 4); err != nil { + return err + } + + v.FeeRate = int32(feeRate) + v.BaseFee = int32(baseFee) + + return nil +} diff --git a/payments/db/migration1/lnwire/update_add_htlc.go b/payments/db/migration1/lnwire/update_add_htlc.go new file mode 100644 index 000000000..38ceeec21 --- /dev/null +++ b/payments/db/migration1/lnwire/update_add_htlc.go @@ -0,0 +1,226 @@ +package lnwire + +import ( + "bytes" + "io" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // OnionPacketSize is the size of the serialized Sphinx onion packet + // included in each UpdateAddHTLC message. The breakdown of the onion + // packet is as follows: 1-byte version, 33-byte ephemeral public key + // (for ECDH), 1300-bytes of per-hop data, and a 32-byte HMAC over the + // entire packet. + OnionPacketSize = 1366 + + // ExperimentalAccountableType is the TLV type used for a custom + // record that sets an experimental accountable value. + ExperimentalAccountableType tlv.Type = 106823 + + // ExperimentalUnaccountable is the value that the experimental + // accountable field contains when a htlc is not accountable. + ExperimentalUnaccountable = 0 + + // ExperimentalAccountable is the value that the experimental + // accountable field contains when a htlc is accountable. We're using a + // single byte to represent our accountable value, but limit the value + // to using the first three bits (max value = 00000111). Interpreted as + // a uint8 (an alias for byte in go), we can just define this constant + // as 7. + ExperimentalAccountable = 7 +) + +type ( + // BlindingPointTlvType is the type for ephemeral pubkeys used in + // route blinding. + BlindingPointTlvType = tlv.TlvType0 + + // BlindingPointRecord holds an optional blinding point on update add + // htlc. + //nolint:ll + BlindingPointRecord = tlv.OptionalRecordT[BlindingPointTlvType, *btcec.PublicKey] +) + +// UpdateAddHTLC is the message sent by Alice to Bob when she wishes to add an +// HTLC to his remote commitment transaction. In addition to information +// detailing the value, the ID, expiry, and the onion blob is also included +// which allows Bob to derive the next hop in the route. The HTLC added by this +// message is to be added to the remote node's "pending" HTLCs. A subsequent +// CommitSig message will move the pending HTLC to the newly created commitment +// transaction, marking them as "staged". +type UpdateAddHTLC struct { + // ChanID is the particular active channel that this UpdateAddHTLC is + // bound to. + ChanID ChannelID + + // ID is the identification server for this HTLC. This value is + // explicitly included as it allows nodes to survive single-sided + // restarts. The ID value for this sides starts at zero, and increases + // with each offered HTLC. + ID uint64 + + // Amount is the amount of millisatoshis this HTLC is worth. + Amount MilliSatoshi + + // PaymentHash is the payment hash to be included in the HTLC this + // request creates. The pre-image to this HTLC must be revealed by the + // upstream peer in order to fully settle the HTLC. + PaymentHash [32]byte + + // Expiry is the number of blocks after which this HTLC should expire. + // It is the receiver's duty to ensure that the outgoing HTLC has a + // sufficient expiry value to allow her to redeem the incoming HTLC. + Expiry uint32 + + // OnionBlob is the raw serialized mix header used to route an HTLC in + // a privacy-preserving manner. The mix header is defined currently to + // be parsed as a 4-tuple: (groupElement, routingInfo, headerMAC, + // body). First the receiving node should use the groupElement, and + // its current onion key to derive a shared secret with the source. + // Once the shared secret has been derived, the headerMAC should be + // checked FIRST. Note that the MAC only covers the routingInfo field. + // If the MAC matches, and the shared secret is fresh, then the node + // should strip off a layer of encryption, exposing the next hop to be + // used in the subsequent UpdateAddHTLC message. + OnionBlob [OnionPacketSize]byte + + // BlindingPoint is the ephemeral pubkey used to optionally blind the + // next hop for this htlc. + BlindingPoint BlindingPointRecord + + // CustomRecords maps TLV types to byte slices, storing arbitrary data + // intended for inclusion in the ExtraData field of the UpdateAddHTLC + // message. + CustomRecords CustomRecords + + // ExtraData is the set of data that was appended to this message to + // fill out the full maximum transport message size. These fields can + // be used to specify optional data such as custom TLV fields. + ExtraData ExtraOpaqueData +} + +// NewUpdateAddHTLC returns a new empty UpdateAddHTLC message. +func NewUpdateAddHTLC() *UpdateAddHTLC { + return &UpdateAddHTLC{} +} + +// A compile time check to ensure UpdateAddHTLC implements the lnwire.Message +// interface. +var _ Message = (*UpdateAddHTLC)(nil) + +// Decode deserializes a serialized UpdateAddHTLC message stored in the passed +// io.Reader observing the specified protocol version. +// +// This is part of the lnwire.Message interface. +func (c *UpdateAddHTLC) Decode(r io.Reader, pver uint32) error { + // msgExtraData is a temporary variable used to read the message extra + // data field from the reader. + var msgExtraData ExtraOpaqueData + + if err := ReadElements(r, + &c.ChanID, + &c.ID, + &c.Amount, + c.PaymentHash[:], + &c.Expiry, + c.OnionBlob[:], + &msgExtraData, + ); err != nil { + return err + } + + // Extract TLV records from the extra data field. + blindingRecord := c.BlindingPoint.Zero() + + customRecords, parsed, extraData, err := ParseAndExtractCustomRecords( + msgExtraData, &blindingRecord, + ) + if err != nil { + return err + } + + // Assign the parsed records back to the message. + if parsed.Contains(blindingRecord.TlvType()) { + c.BlindingPoint = tlv.SomeRecordT(blindingRecord) + } + + c.CustomRecords = customRecords + c.ExtraData = extraData + + return nil +} + +// Encode serializes the target UpdateAddHTLC into the passed io.Writer +// observing the protocol version specified. +// +// This is part of the lnwire.Message interface. +func (c *UpdateAddHTLC) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteChannelID(w, c.ChanID); err != nil { + return err + } + + if err := WriteUint64(w, c.ID); err != nil { + return err + } + + if err := WriteMilliSatoshi(w, c.Amount); err != nil { + return err + } + + if err := WriteBytes(w, c.PaymentHash[:]); err != nil { + return err + } + + if err := WriteUint32(w, c.Expiry); err != nil { + return err + } + + if err := WriteBytes(w, c.OnionBlob[:]); err != nil { + return err + } + + // Only include blinding point in extra data if present. + var records []tlv.RecordProducer + c.BlindingPoint.WhenSome( + func(b tlv.RecordT[BlindingPointTlvType, *btcec.PublicKey]) { + records = append(records, &b) + }, + ) + + extraData, err := MergeAndEncode(records, c.ExtraData, c.CustomRecords) + if err != nil { + return err + } + + return WriteBytes(w, extraData) +} + +// MsgType returns the integer uniquely identifying this message type on the +// wire. +// +// This is part of the lnwire.Message interface. +func (c *UpdateAddHTLC) MsgType() MessageType { + return MsgUpdateAddHTLC +} + +// TargetChanID returns the channel id of the link for which this message is +// intended. +// +// NOTE: Part of peer.LinkUpdater interface. +func (c *UpdateAddHTLC) TargetChanID() ChannelID { + return c.ChanID +} + +// SerializedSize returns the serialized size of the message in bytes. +// +// This is part of the lnwire.SizeableMessage interface. +func (c *UpdateAddHTLC) SerializedSize() (uint32, error) { + return MessageSerializedSize(c) +} + +// A compile time check to ensure UpdateAddHTLC implements the +// lnwire.SizeableMessage interface. +var _ SizeableMessage = (*UpdateAddHTLC)(nil) diff --git a/payments/db/migration1/lnwire/writer.go b/payments/db/migration1/lnwire/writer.go new file mode 100644 index 000000000..db575663e --- /dev/null +++ b/payments/db/migration1/lnwire/writer.go @@ -0,0 +1,185 @@ +package lnwire + +import ( + "bytes" + "encoding/binary" + "errors" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/btcutil" +) + +var ( + // ErrNilFeatureVector is returned when the supplied feature is nil. + ErrNilFeatureVector = errors.New("cannot write nil feature vector") + + // ErrNilPublicKey is returned when a nil pubkey is used. + ErrNilPublicKey = errors.New("cannot write nil pubkey") +) + +// WriteBytes appends the given bytes to the provided buffer. +func WriteBytes(buf *bytes.Buffer, b []byte) error { + _, err := buf.Write(b) + return err +} + +// WriteUint8 appends the uint8 to the provided buffer. +func WriteUint8(buf *bytes.Buffer, n uint8) error { + _, err := buf.Write([]byte{n}) + return err +} + +// WriteUint16 appends the uint16 to the provided buffer. It encodes the +// integer using big endian byte order. +func WriteUint16(buf *bytes.Buffer, n uint16) error { + var b [2]byte + binary.BigEndian.PutUint16(b[:], n) + _, err := buf.Write(b[:]) + return err +} + +// WriteUint32 appends the uint32 to the provided buffer. It encodes the +// integer using big endian byte order. +func WriteUint32(buf *bytes.Buffer, n uint32) error { + var b [4]byte + binary.BigEndian.PutUint32(b[:], n) + _, err := buf.Write(b[:]) + return err +} + +// WriteUint64 appends the uint64 to the provided buffer. It encodes the +// integer using big endian byte order. +func WriteUint64(buf *bytes.Buffer, n uint64) error { + var b [8]byte + binary.BigEndian.PutUint64(b[:], n) + _, err := buf.Write(b[:]) + return err +} + +// WriteSatoshi appends the Satoshi value to the provided buffer. +func WriteSatoshi(buf *bytes.Buffer, amount btcutil.Amount) error { + return WriteUint64(buf, uint64(amount)) +} + +// WriteMilliSatoshi appends the MilliSatoshi value to the provided buffer. +func WriteMilliSatoshi(buf *bytes.Buffer, amount MilliSatoshi) error { + return WriteUint64(buf, uint64(amount)) +} + +// WritePublicKey appends the compressed public key to the provided buffer. +func WritePublicKey(buf *bytes.Buffer, pub *btcec.PublicKey) error { + if pub == nil { + return ErrNilPublicKey + } + + serializedPubkey := pub.SerializeCompressed() + return WriteBytes(buf, serializedPubkey) +} + +// WriteChannelID appends the ChannelID to the provided buffer. +func WriteChannelID(buf *bytes.Buffer, channelID ChannelID) error { + return WriteBytes(buf, channelID[:]) +} + +// WriteShortChannelID appends the ShortChannelID to the provided buffer. It +// encodes the BlockHeight and TxIndex each using 3 bytes with big endian byte +// order, and encodes txPosition using 2 bytes with big endian byte order. +func WriteShortChannelID(buf *bytes.Buffer, shortChanID ShortChannelID) error { + // Check that field fit in 3 bytes and write the blockHeight + if shortChanID.BlockHeight > ((1 << 24) - 1) { + return errors.New("block height should fit in 3 bytes") + } + + var blockHeight [4]byte + binary.BigEndian.PutUint32(blockHeight[:], shortChanID.BlockHeight) + + if _, err := buf.Write(blockHeight[1:]); err != nil { + return err + } + + // Check that field fit in 3 bytes and write the txIndex + if shortChanID.TxIndex > ((1 << 24) - 1) { + return errors.New("tx index should fit in 3 bytes") + } + + var txIndex [4]byte + binary.BigEndian.PutUint32(txIndex[:], shortChanID.TxIndex) + if _, err := buf.Write(txIndex[1:]); err != nil { + return err + } + + // Write the TxPosition + return WriteUint16(buf, shortChanID.TxPosition) +} + +// WriteSig appends the signature to the provided buffer. +func WriteSig(buf *bytes.Buffer, sig Sig) error { + return WriteBytes(buf, sig.bytes[:]) +} + +// WriteSigs appends the slice of signatures to the provided buffer with its +// length. +func WriteSigs(buf *bytes.Buffer, sigs []Sig) error { + // Write the length of the sigs. + if err := WriteUint16(buf, uint16(len(sigs))); err != nil { + return err + } + + for _, sig := range sigs { + if err := WriteSig(buf, sig); err != nil { + return err + } + } + return nil +} + +// WriteFailCode appends the FailCode to the provided buffer. +func WriteFailCode(buf *bytes.Buffer, e FailCode) error { + return WriteUint16(buf, uint16(e)) +} + +// WriteRawFeatureVector encodes the feature using the feature's Encode method +// and appends the data to the provided buffer. An error will return if the +// passed feature is nil. +func WriteRawFeatureVector(buf *bytes.Buffer, feature *RawFeatureVector) error { + if feature == nil { + return ErrNilFeatureVector + } + + return feature.Encode(buf) +} + +// WriteChanUpdateMsgFlags appends the update flag to the provided buffer. +func WriteChanUpdateMsgFlags(buf *bytes.Buffer, f ChanUpdateMsgFlags) error { + return WriteUint8(buf, uint8(f)) +} + +// WriteChanUpdateChanFlags appends the update flag to the provided buffer. +func WriteChanUpdateChanFlags(buf *bytes.Buffer, f ChanUpdateChanFlags) error { + return WriteUint8(buf, uint8(f)) +} + +// WriteErrorData appends the data to the provided buffer. +func WriteErrorData(buf *bytes.Buffer, data ErrorData) error { + return writeDataWithLength(buf, data) +} + +// WriteBool appends the boolean to the provided buffer. +func WriteBool(buf *bytes.Buffer, b bool) error { + if b { + return WriteBytes(buf, []byte{1}) + } + return WriteBytes(buf, []byte{0}) +} + +// writeDataWithLength writes the data and its length to the buffer. +func writeDataWithLength(buf *bytes.Buffer, data []byte) error { + var l [2]byte + binary.BigEndian.PutUint16(l[:], uint16(len(data))) + if _, err := buf.Write(l[:]); err != nil { + return err + } + + _, err := buf.Write(data) + return err +} diff --git a/payments/db/migration1/migration_validation.go b/payments/db/migration1/migration_validation.go index fe43cf28f..f20c92783 100644 --- a/payments/db/migration1/migration_validation.go +++ b/payments/db/migration1/migration_validation.go @@ -12,9 +12,9 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/lightningnetwork/lnd/kvdb" "github.com/lightningnetwork/lnd/lntypes" - "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/record" "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" - "github.com/lightningnetwork/lnd/record" "github.com/pmezard/go-difflib/difflib" ) diff --git a/payments/db/migration1/payment.go b/payments/db/migration1/payment.go index 53a3d814e..78599c3ed 100644 --- a/payments/db/migration1/payment.go +++ b/payments/db/migration1/payment.go @@ -8,7 +8,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" sphinx "github.com/lightningnetwork/lightning-onion" "github.com/lightningnetwork/lnd/lntypes" - "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" ) // FailureReason encodes the reason a payment ultimately failed. diff --git a/payments/db/migration1/record/amp.go b/payments/db/migration1/record/amp.go new file mode 100644 index 000000000..f63c7a141 --- /dev/null +++ b/payments/db/migration1/record/amp.go @@ -0,0 +1,121 @@ +package record + +import ( + "fmt" + "io" + + "github.com/lightningnetwork/lnd/tlv" +) + +// AMPOnionType is the type used in the onion to reference the AMP fields: +// root_share, set_id, and child_index. +const AMPOnionType tlv.Type = 14 + +// AMP is a record that encodes the fields necessary for atomic multi-path +// payments. +type AMP struct { + rootShare [32]byte + setID [32]byte + childIndex uint32 +} + +// MaxAmpPayLoadSize is an AMP Record which when serialized to a tlv record uses +// the maximum payload size. The `childIndex` is created randomly and is a +// 4 byte `varint` type so we make sure we use an index which will be encoded in +// 4 bytes. +var MaxAmpPayLoadSize = AMP{ + rootShare: [32]byte{}, + setID: [32]byte{}, + childIndex: 0x80000000, +} + +// NewAMP generate a new AMP record with the given root_share, set_id, and +// child_index. +func NewAMP(rootShare, setID [32]byte, childIndex uint32) *AMP { + return &{ + rootShare: rootShare, + setID: setID, + childIndex: childIndex, + } +} + +// RootShare returns the root share contained in the AMP record. +func (a *AMP) RootShare() [32]byte { + return a.rootShare +} + +// SetID returns the set id contained in the AMP record. +func (a *AMP) SetID() [32]byte { + return a.setID +} + +// ChildIndex returns the child index contained in the AMP record. +func (a *AMP) ChildIndex() uint32 { + return a.childIndex +} + +// AMPEncoder writes the AMP record to the provided io.Writer. +func AMPEncoder(w io.Writer, val interface{}, buf *[8]byte) error { + if v, ok := val.(*AMP); ok { + if err := tlv.EBytes32(w, &v.rootShare, buf); err != nil { + return err + } + + if err := tlv.EBytes32(w, &v.setID, buf); err != nil { + return err + } + + return tlv.ETUint32T(w, v.childIndex, buf) + } + return tlv.NewTypeForEncodingErr(val, "AMP") +} + +const ( + // minAMPLength is the minimum length of a serialized AMP TLV record, + // which occurs when the truncated encoding of child_index takes 0 + // bytes, leaving only the root_share and set_id. + minAMPLength = 64 + + // maxAMPLength is the maximum length of a serialized AMP TLV record, + // which occurs when the truncated encoding of a child_index takes 2 + // bytes. + maxAMPLength = 68 +) + +// AMPDecoder reads the AMP record from the provided io.Reader. +func AMPDecoder(r io.Reader, val interface{}, buf *[8]byte, l uint64) error { + if v, ok := val.(*AMP); ok && minAMPLength <= l && l <= maxAMPLength { + if err := tlv.DBytes32(r, &v.rootShare, buf, 32); err != nil { + return err + } + + if err := tlv.DBytes32(r, &v.setID, buf, 32); err != nil { + return err + } + + return tlv.DTUint32(r, &v.childIndex, buf, l-minAMPLength) + } + return tlv.NewTypeForDecodingErr(val, "AMP", l, maxAMPLength) +} + +// Record returns a tlv.Record that can be used to encode or decode this record. +func (a *AMP) Record() tlv.Record { + return tlv.MakeDynamicRecord( + AMPOnionType, a, a.PayloadSize, AMPEncoder, AMPDecoder, + ) +} + +// PayloadSize returns the size this record takes up in encoded form. +func (a *AMP) PayloadSize() uint64 { + return 32 + 32 + tlv.SizeTUint32(a.childIndex) +} + +// String returns a human-readable description of the amp payload fields. +func (a *AMP) String() string { + if a == nil { + return "" + } + + return fmt.Sprintf("root_share=%x set_id=%x child_index=%d", + a.rootShare, a.setID, a.childIndex) +} diff --git a/payments/db/migration1/record/blinded_data.go b/payments/db/migration1/record/blinded_data.go new file mode 100644 index 000000000..22c09672f --- /dev/null +++ b/payments/db/migration1/record/blinded_data.go @@ -0,0 +1,440 @@ +package record + +import ( + "bytes" + "encoding/binary" + "io" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/tlv" +) + +// AverageDummyHopPayloadSize is the size of a standard blinded path dummy hop +// payload. In most cases, this is larger than the other payload types and so +// to make sure that a sender cannot use this fact to know if a dummy hop is +// present or not, we'll make sure to always pad all payloads to at least this +// size. +const AverageDummyHopPayloadSize = 51 + +// BlindedRouteData contains the information that is included in a blinded +// route encrypted data blob that is created by the recipient to provide +// forwarding information. +type BlindedRouteData struct { + // Padding is an optional set of bytes that a recipient can use to pad + // the data so that the encrypted recipient data blobs are all the same + // length. + Padding tlv.OptionalRecordT[tlv.TlvType1, []byte] + + // ShortChannelID is the channel ID of the next hop. + ShortChannelID tlv.OptionalRecordT[tlv.TlvType2, lnwire.ShortChannelID] + + // NextNodeID is the node ID of the next node on the path. In the + // context of blinded path payments, this is used to indicate the + // presence of dummy hops that need to be peeled from the onion. + NextNodeID tlv.OptionalRecordT[tlv.TlvType4, *btcec.PublicKey] + + // PathID is a secret set of bytes that the blinded path creator will + // set so that they can check the value on decryption to ensure that the + // path they created was used for the intended purpose. + PathID tlv.OptionalRecordT[tlv.TlvType6, []byte] + + // NextBlindingOverride is a blinding point that should be switched + // in for the next hop. This is used to combine two blinded paths into + // one (which primarily is used in onion messaging, but in theory + // could be used for payments as well). + NextBlindingOverride tlv.OptionalRecordT[tlv.TlvType8, *btcec.PublicKey] + + // RelayInfo provides the relay parameters for the hop. + RelayInfo tlv.OptionalRecordT[tlv.TlvType10, PaymentRelayInfo] + + // Constraints provides the payment relay constraints for the hop. + Constraints tlv.OptionalRecordT[tlv.TlvType12, PaymentConstraints] + + // Features is the set of features the payment requires. + Features tlv.OptionalRecordT[tlv.TlvType14, lnwire.FeatureVector] +} + +// NewNonFinalBlindedRouteData creates the data that's provided for hops within +// a blinded route. +func NewNonFinalBlindedRouteData(chanID lnwire.ShortChannelID, + blindingOverride *btcec.PublicKey, relayInfo PaymentRelayInfo, + constraints *PaymentConstraints, + features *lnwire.FeatureVector) *BlindedRouteData { + + info := &BlindedRouteData{ + ShortChannelID: tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType2](chanID), + ), + RelayInfo: tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType10](relayInfo), + ), + } + + if blindingOverride != nil { + info.NextBlindingOverride = tlv.SomeRecordT( + tlv.NewPrimitiveRecord[tlv.TlvType8](blindingOverride)) + } + + if constraints != nil { + info.Constraints = tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType12](*constraints)) + } + + if features != nil { + info.Features = tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType14](*features), + ) + } + + return info +} + +// NewFinalHopBlindedRouteData creates the data that's provided for the final +// hop in a blinded route. +func NewFinalHopBlindedRouteData(constraints *PaymentConstraints, + pathID []byte) *BlindedRouteData { + + var data BlindedRouteData + if pathID != nil { + data.PathID = tlv.SomeRecordT( + tlv.NewPrimitiveRecord[tlv.TlvType6](pathID), + ) + } + + if constraints != nil { + data.Constraints = tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType12](*constraints)) + } + + return &data +} + +// NewDummyHopRouteData creates the data that's provided for any hop preceding +// a dummy hop. The presence of such a payload indicates to the reader that +// they are the intended recipient and should peel the remainder of the onion. +func NewDummyHopRouteData(ourPubKey *btcec.PublicKey, + relayInfo PaymentRelayInfo, + constraints PaymentConstraints) *BlindedRouteData { + + return &BlindedRouteData{ + NextNodeID: tlv.SomeRecordT( + tlv.NewPrimitiveRecord[tlv.TlvType4](ourPubKey), + ), + RelayInfo: tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType10](relayInfo), + ), + Constraints: tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType12](constraints), + ), + } +} + +// DecodeBlindedRouteData decodes the data provided within a blinded route. +func DecodeBlindedRouteData(r io.Reader) (*BlindedRouteData, error) { + var ( + d BlindedRouteData + + padding = d.Padding.Zero() + scid = d.ShortChannelID.Zero() + nextNodeID = d.NextNodeID.Zero() + pathID = d.PathID.Zero() + blindingOverride = d.NextBlindingOverride.Zero() + relayInfo = d.RelayInfo.Zero() + constraints = d.Constraints.Zero() + features = d.Features.Zero() + ) + + var tlvRecords lnwire.ExtraOpaqueData + if err := lnwire.ReadElements(r, &tlvRecords); err != nil { + return nil, err + } + + typeMap, err := tlvRecords.ExtractRecords( + &padding, &scid, &nextNodeID, &pathID, &blindingOverride, + &relayInfo, &constraints, &features, + ) + if err != nil { + return nil, err + } + + val, ok := typeMap[d.Padding.TlvType()] + if ok && val == nil { + d.Padding = tlv.SomeRecordT(padding) + } + + if val, ok := typeMap[d.ShortChannelID.TlvType()]; ok && val == nil { + d.ShortChannelID = tlv.SomeRecordT(scid) + } + + if val, ok := typeMap[d.NextNodeID.TlvType()]; ok && val == nil { + d.NextNodeID = tlv.SomeRecordT(nextNodeID) + } + + if val, ok := typeMap[d.PathID.TlvType()]; ok && val == nil { + d.PathID = tlv.SomeRecordT(pathID) + } + + val, ok = typeMap[d.NextBlindingOverride.TlvType()] + if ok && val == nil { + d.NextBlindingOverride = tlv.SomeRecordT(blindingOverride) + } + + if val, ok := typeMap[d.RelayInfo.TlvType()]; ok && val == nil { + d.RelayInfo = tlv.SomeRecordT(relayInfo) + } + + if val, ok := typeMap[d.Constraints.TlvType()]; ok && val == nil { + d.Constraints = tlv.SomeRecordT(constraints) + } + + if val, ok := typeMap[d.Features.TlvType()]; ok && val == nil { + d.Features = tlv.SomeRecordT(features) + } + + return &d, nil +} + +// EncodeBlindedRouteData encodes the blinded route data provided. +func EncodeBlindedRouteData(data *BlindedRouteData) ([]byte, error) { + var ( + e lnwire.ExtraOpaqueData + recordProducers = make([]tlv.RecordProducer, 0, 5) + ) + + data.Padding.WhenSome(func(p tlv.RecordT[tlv.TlvType1, []byte]) { + recordProducers = append(recordProducers, &p) + }) + + data.ShortChannelID.WhenSome(func(scid tlv.RecordT[tlv.TlvType2, + lnwire.ShortChannelID]) { + + recordProducers = append(recordProducers, &scid) + }) + + data.NextNodeID.WhenSome(func(f tlv.RecordT[tlv.TlvType4, + *btcec.PublicKey]) { + + recordProducers = append(recordProducers, &f) + }) + + data.PathID.WhenSome(func(pathID tlv.RecordT[tlv.TlvType6, []byte]) { + recordProducers = append(recordProducers, &pathID) + }) + + data.NextBlindingOverride.WhenSome(func(pk tlv.RecordT[tlv.TlvType8, + *btcec.PublicKey]) { + + recordProducers = append(recordProducers, &pk) + }) + + data.RelayInfo.WhenSome(func(r tlv.RecordT[tlv.TlvType10, + PaymentRelayInfo]) { + + recordProducers = append(recordProducers, &r) + }) + + data.Constraints.WhenSome(func(cs tlv.RecordT[tlv.TlvType12, + PaymentConstraints]) { + + recordProducers = append(recordProducers, &cs) + }) + + data.Features.WhenSome(func(f tlv.RecordT[tlv.TlvType14, + lnwire.FeatureVector]) { + + recordProducers = append(recordProducers, &f) + }) + + if err := e.PackRecords(recordProducers...); err != nil { + return nil, err + } + + return e[:], nil +} + +// PadBy adds "n" padding bytes to the BlindedRouteData using the Padding field. +// Callers should be aware that the total payload size will change by more than +// "n" since the "n" bytes will be prefixed by BigSize type and length fields. +// Callers may need to call PadBy iteratively until each encrypted data packet +// is the same size and so each call will overwrite the Padding record. +// Note that calling PadBy with an n value of 0 will still result in a zero +// length TLV entry being added. +func (b *BlindedRouteData) PadBy(n int) { + b.Padding = tlv.SomeRecordT( + tlv.NewPrimitiveRecord[tlv.TlvType1](make([]byte, n)), + ) +} + +// PaymentRelayInfo describes the relay policy for a blinded path. +type PaymentRelayInfo struct { + // CltvExpiryDelta is the expiry delta for the payment. + CltvExpiryDelta uint16 + + // FeeRate is the fee rate that will be charged per millionth of a + // satoshi. + FeeRate uint32 + + // BaseFee is the per-htlc fee charged in milli-satoshis. + BaseFee lnwire.MilliSatoshi +} + +// Record creates a tlv.Record that encodes the payment relay (type 10) type for +// an encrypted blob payload. +func (i *PaymentRelayInfo) Record() tlv.Record { + return tlv.MakeDynamicRecord( + 10, &i, func() uint64 { + // uint16 + uint32 + tuint32 + return 2 + 4 + tlv.SizeTUint32(uint32(i.BaseFee)) + }, encodePaymentRelay, decodePaymentRelay, + ) +} + +func encodePaymentRelay(w io.Writer, val interface{}, buf *[8]byte) error { + if t, ok := val.(**PaymentRelayInfo); ok { + relayInfo := *t + + // Just write our first 6 bytes directly. + binary.BigEndian.PutUint16(buf[:2], relayInfo.CltvExpiryDelta) + binary.BigEndian.PutUint32(buf[2:6], relayInfo.FeeRate) + if _, err := w.Write(buf[0:6]); err != nil { + return err + } + + baseFee := uint32(relayInfo.BaseFee) + + // We can safely reuse buf here because we overwrite its + // contents. + return tlv.ETUint32(w, &baseFee, buf) + } + + return tlv.NewTypeForEncodingErr(val, "**hop.PaymentRelayInfo") +} + +func decodePaymentRelay(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if t, ok := val.(**PaymentRelayInfo); ok && l <= 10 { + scratch := make([]byte, l) + + n, err := io.ReadFull(r, scratch) + if err != nil { + return err + } + + // We expect at least 6 bytes, because we have 2 bytes for + // cltv delta and 4 bytes for fee rate. + if n < 6 { + return tlv.NewTypeForDecodingErr(val, + "*hop.paymentRelayInfo", uint64(n), 6) + } + + relayInfo := *t + + relayInfo.CltvExpiryDelta = binary.BigEndian.Uint16( + scratch[0:2], + ) + relayInfo.FeeRate = binary.BigEndian.Uint32(scratch[2:6]) + + // To be able to re-use the DTUint32 function we create a + // buffer with just the bytes holding the variable length u32. + // If the base fee is zero, this will be an empty buffer, which + // is okay. + b := bytes.NewBuffer(scratch[6:]) + + var baseFee uint32 + err = tlv.DTUint32(b, &baseFee, buf, l-6) + if err != nil { + return err + } + + relayInfo.BaseFee = lnwire.MilliSatoshi(baseFee) + + return nil + } + + return tlv.NewTypeForDecodingErr(val, "*hop.paymentRelayInfo", l, 10) +} + +// PaymentConstraints is a set of restrictions on a payment. +type PaymentConstraints struct { + // MaxCltvExpiry is the maximum expiry height for the payment. + MaxCltvExpiry uint32 + + // HtlcMinimumMsat is the minimum htlc size for the payment. + HtlcMinimumMsat lnwire.MilliSatoshi +} + +func (p *PaymentConstraints) Record() tlv.Record { + return tlv.MakeDynamicRecord( + 12, &p, func() uint64 { + // uint32 + tuint64. + return 4 + tlv.SizeTUint64(uint64( + p.HtlcMinimumMsat, + )) + }, + encodePaymentConstraints, decodePaymentConstraints, + ) +} + +func encodePaymentConstraints(w io.Writer, val interface{}, + buf *[8]byte) error { + + if c, ok := val.(**PaymentConstraints); ok { + constraints := *c + + binary.BigEndian.PutUint32(buf[:4], constraints.MaxCltvExpiry) + if _, err := w.Write(buf[:4]); err != nil { + return err + } + + // We can safely re-use buf here because we overwrite its + // contents. + htlcMsat := uint64(constraints.HtlcMinimumMsat) + + return tlv.ETUint64(w, &htlcMsat, buf) + } + + return tlv.NewTypeForEncodingErr(val, "**PaymentConstraints") +} + +func decodePaymentConstraints(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if c, ok := val.(**PaymentConstraints); ok && l <= 12 { + scratch := make([]byte, l) + + n, err := io.ReadFull(r, scratch) + if err != nil { + return err + } + + // We expect at least 4 bytes for our uint32. + if n < 4 { + return tlv.NewTypeForDecodingErr(val, + "*paymentConstraints", uint64(n), 4) + } + + payConstraints := *c + + payConstraints.MaxCltvExpiry = binary.BigEndian.Uint32( + scratch[:4], + ) + + // This could be empty if our minimum is zero, that's okay. + var ( + b = bytes.NewBuffer(scratch[4:]) + minHtlc uint64 + ) + + err = tlv.DTUint64(b, &minHtlc, buf, l-4) + if err != nil { + return err + } + payConstraints.HtlcMinimumMsat = lnwire.MilliSatoshi(minHtlc) + + return nil + } + + return tlv.NewTypeForDecodingErr(val, "**PaymentConstraints", l, l) +} diff --git a/payments/db/migration1/record/custom_records.go b/payments/db/migration1/record/custom_records.go new file mode 100644 index 000000000..01952c24e --- /dev/null +++ b/payments/db/migration1/record/custom_records.go @@ -0,0 +1,31 @@ +package record + +import ( + "fmt" +) + +const ( + // CustomTypeStart is the start of the custom tlv type range as defined + // in BOLT 01. + CustomTypeStart = 65536 +) + +// CustomSet stores a set of custom key/value pairs. +type CustomSet map[uint64][]byte + +// Validate checks that all custom records are in the custom type range. +func (c CustomSet) Validate() error { + for key := range c { + if key < CustomTypeStart { + return fmt.Errorf("no custom records with types "+ + "below %v allowed", CustomTypeStart) + } + } + + return nil +} + +// IsKeysend checks if the custom records contain the key send type. +func (c CustomSet) IsKeysend() bool { + return c[KeySendType] != nil +} diff --git a/payments/db/migration1/record/experimental.go b/payments/db/migration1/record/experimental.go new file mode 100644 index 000000000..3aff0ff26 --- /dev/null +++ b/payments/db/migration1/record/experimental.go @@ -0,0 +1,6 @@ +package record + +const ( + // KeySendType is the custom record identifier for keysend preimages. + KeySendType uint64 = 5482373484 +) diff --git a/payments/db/migration1/record/hop.go b/payments/db/migration1/record/hop.go new file mode 100644 index 000000000..e5c0884f1 --- /dev/null +++ b/payments/db/migration1/record/hop.go @@ -0,0 +1,99 @@ +package record + +import ( + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // AmtOnionType is the type used in the onion to reference the amount to + // send to the next hop. + AmtOnionType tlv.Type = 2 + + // LockTimeTLV is the type used in the onion to reference the CLTV + // value that should be used for the next hop's HTLC. + LockTimeOnionType tlv.Type = 4 + + // NextHopOnionType is the type used in the onion to reference the ID + // of the next hop. + NextHopOnionType tlv.Type = 6 + + // EncryptedDataOnionType is the type used to include encrypted data + // provided by the receiver in the onion for use in blinded paths. + EncryptedDataOnionType tlv.Type = 10 + + // BlindingPointOnionType is the type used to include receiver provided + // ephemeral keys in the onion that are used in blinded paths. + BlindingPointOnionType tlv.Type = 12 + + // MetadataOnionType is the type used in the onion for the payment + // metadata. + MetadataOnionType tlv.Type = 16 + + // TotalAmtMsatBlindedType is the type used in the onion for the total + // amount field that is included in the final hop for blinded payments. + TotalAmtMsatBlindedType tlv.Type = 18 +) + +// NewAmtToFwdRecord creates a tlv.Record that encodes the amount_to_forward +// (type 2) for an onion payload. +func NewAmtToFwdRecord(amt *uint64) tlv.Record { + return tlv.MakeDynamicRecord( + AmtOnionType, amt, func() uint64 { + return tlv.SizeTUint64(*amt) + }, + tlv.ETUint64, tlv.DTUint64, + ) +} + +// NewLockTimeRecord creates a tlv.Record that encodes the outgoing_cltv_value +// (type 4) for an onion payload. +func NewLockTimeRecord(lockTime *uint32) tlv.Record { + return tlv.MakeDynamicRecord( + LockTimeOnionType, lockTime, func() uint64 { + return tlv.SizeTUint32(*lockTime) + }, + tlv.ETUint32, tlv.DTUint32, + ) +} + +// NewNextHopIDRecord creates a tlv.Record that encodes the short_channel_id +// (type 6) for an onion payload. +func NewNextHopIDRecord(cid *uint64) tlv.Record { + return tlv.MakePrimitiveRecord(NextHopOnionType, cid) +} + +// NewEncryptedDataRecord creates a tlv.Record that encodes the encrypted_data +// (type 10) record for an onion payload. +func NewEncryptedDataRecord(data *[]byte) tlv.Record { + return tlv.MakePrimitiveRecord(EncryptedDataOnionType, data) +} + +// NewBlindingPointRecord creates a tlv.Record that encodes the blinding_point +// (type 12) record for an onion payload. +func NewBlindingPointRecord(point **btcec.PublicKey) tlv.Record { + return tlv.MakePrimitiveRecord(BlindingPointOnionType, point) +} + +// NewMetadataRecord creates a tlv.Record that encodes the metadata (type 10) +// for an onion payload. +func NewMetadataRecord(metadata *[]byte) tlv.Record { + return tlv.MakeDynamicRecord( + MetadataOnionType, metadata, + func() uint64 { + return uint64(len(*metadata)) + }, + tlv.EVarBytes, tlv.DVarBytes, + ) +} + +// NewTotalAmtMsatBlinded creates a tlv.Record that encodes the +// total_amount_msat for the final an onion payload within a blinded route. +func NewTotalAmtMsatBlinded(amt *uint64) tlv.Record { + return tlv.MakeDynamicRecord( + TotalAmtMsatBlindedType, amt, func() uint64 { + return tlv.SizeTUint64(*amt) + }, + tlv.ETUint64, tlv.DTUint64, + ) +} diff --git a/payments/db/migration1/record/mpp.go b/payments/db/migration1/record/mpp.go new file mode 100644 index 000000000..576ada776 --- /dev/null +++ b/payments/db/migration1/record/mpp.go @@ -0,0 +1,112 @@ +package record + +import ( + "fmt" + "io" + + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/tlv" +) + +// MPPOnionType is the type used in the onion to reference the MPP fields: +// total_amt and payment_addr. +const MPPOnionType tlv.Type = 8 + +// MPP is a record that encodes the fields necessary for multi-path payments. +type MPP struct { + // paymentAddr is a random, receiver-generated value used to avoid + // collisions with concurrent payers. + paymentAddr [32]byte + + // totalMsat is the total value of the payment, potentially spread + // across more than one HTLC. + totalMsat lnwire.MilliSatoshi +} + +// NewMPP generates a new MPP record with the given total and payment address. +func NewMPP(total lnwire.MilliSatoshi, addr [32]byte) *MPP { + return &MPP{ + paymentAddr: addr, + totalMsat: total, + } +} + +// PaymentAddr returns the payment address contained in the MPP record. +func (r *MPP) PaymentAddr() [32]byte { + return r.paymentAddr +} + +// TotalMsat returns the total value of an MPP payment in msats. +func (r *MPP) TotalMsat() lnwire.MilliSatoshi { + return r.totalMsat +} + +// MPPEncoder writes the MPP record to the provided io.Writer. +func MPPEncoder(w io.Writer, val interface{}, buf *[8]byte) error { + if v, ok := val.(*MPP); ok { + err := tlv.EBytes32(w, &v.paymentAddr, buf) + if err != nil { + return err + } + + return tlv.ETUint64T(w, uint64(v.totalMsat), buf) + } + return tlv.NewTypeForEncodingErr(val, "MPP") +} + +const ( + // minMPPLength is the minimum length of a serialized MPP TLV record, + // which occurs when the truncated encoding of total_amt_msat takes 0 + // bytes, leaving only the payment_addr. + minMPPLength = 32 + + // maxMPPLength is the maximum length of a serialized MPP TLV record, + // which occurs when the truncated encoding of total_amt_msat takes 8 + // bytes. + maxMPPLength = 40 +) + +// MPPDecoder reads the MPP record to the provided io.Reader. +func MPPDecoder(r io.Reader, val interface{}, buf *[8]byte, l uint64) error { + if v, ok := val.(*MPP); ok && minMPPLength <= l && l <= maxMPPLength { + if err := tlv.DBytes32(r, &v.paymentAddr, buf, 32); err != nil { + return err + } + + var total uint64 + if err := tlv.DTUint64(r, &total, buf, l-32); err != nil { + return err + } + v.totalMsat = lnwire.MilliSatoshi(total) + + return nil + } + return tlv.NewTypeForDecodingErr(val, "MPP", l, maxMPPLength) +} + +// Record returns a tlv.Record that can be used to encode or decode this record. +func (r *MPP) Record() tlv.Record { + // Fixed-size, 32 byte payment address followed by truncated 64-bit + // total msat. + size := func() uint64 { + return 32 + tlv.SizeTUint64(uint64(r.totalMsat)) + } + + return tlv.MakeDynamicRecord( + MPPOnionType, r, size, MPPEncoder, MPPDecoder, + ) +} + +// PayloadSize returns the size this record takes up in encoded form. +func (r *MPP) PayloadSize() uint64 { + return 32 + tlv.SizeTUint64(uint64(r.totalMsat)) +} + +// String returns a human-readable representation of the mpp payload field. +func (r *MPP) String() string { + if r == nil { + return "" + } + + return fmt.Sprintf("total=%v, addr=%x", r.totalMsat, r.paymentAddr) +} diff --git a/payments/db/migration1/route.go b/payments/db/migration1/route.go index dc9a22162..bdf8c6ff0 100644 --- a/payments/db/migration1/route.go +++ b/payments/db/migration1/route.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/btcsuite/btcd/btcec/v2" - "github.com/lightningnetwork/lnd/lnwire" - "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/record" "github.com/lightningnetwork/lnd/tlv" ) diff --git a/payments/db/migration1/sql_converters.go b/payments/db/migration1/sql_converters.go index dbf88b2ea..50d129bf7 100644 --- a/payments/db/migration1/sql_converters.go +++ b/payments/db/migration1/sql_converters.go @@ -8,9 +8,9 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/lightningnetwork/lnd/lntypes" - "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/record" "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" - "github.com/lightningnetwork/lnd/record" "github.com/lightningnetwork/lnd/tlv" ) diff --git a/payments/db/migration1/sql_migration.go b/payments/db/migration1/sql_migration.go index b20811760..2bda95f57 100644 --- a/payments/db/migration1/sql_migration.go +++ b/payments/db/migration1/sql_migration.go @@ -10,7 +10,7 @@ import ( "github.com/lightningnetwork/lnd/kvdb" "github.com/lightningnetwork/lnd/lntypes" - "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" "golang.org/x/time/rate" ) diff --git a/payments/db/migration1/sql_migration_test.go b/payments/db/migration1/sql_migration_test.go index f19dd9179..ab0ee6cc1 100644 --- a/payments/db/migration1/sql_migration_test.go +++ b/payments/db/migration1/sql_migration_test.go @@ -15,9 +15,9 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/lightningnetwork/lnd/kvdb" "github.com/lightningnetwork/lnd/lntypes" - "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/record" "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" - "github.com/lightningnetwork/lnd/record" "github.com/lightningnetwork/lnd/sqldb" "github.com/stretchr/testify/require" ) diff --git a/payments/db/migration1/sql_store.go b/payments/db/migration1/sql_store.go index 2a2c97752..2129656d9 100644 --- a/payments/db/migration1/sql_store.go +++ b/payments/db/migration1/sql_store.go @@ -11,7 +11,7 @@ import ( "time" "github.com/lightningnetwork/lnd/lntypes" - "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" "github.com/lightningnetwork/lnd/sqldb" ) From 89f046a129e9b2b5859fcb341f825e05b62fdaec Mon Sep 17 00:00:00 2001 From: ziggie Date: Tue, 24 Feb 2026 12:59:59 +0100 Subject: [PATCH 16/16] docs: add release notes for KV-to-SQL payment migration --- docs/release-notes/release-notes-0.21.0.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/release-notes/release-notes-0.21.0.md b/docs/release-notes/release-notes-0.21.0.md index 0679bb98b..5c456d2c4 100644 --- a/docs/release-notes/release-notes-0.21.0.md +++ b/docs/release-notes/release-notes-0.21.0.md @@ -217,6 +217,10 @@ db functions Part 2](https://github.com/lightningnetwork/lnd/pull/10308) * [Finalize SQL implementation for payments db](https://github.com/lightningnetwork/lnd/pull/10373) + * [Add the KV-to-SQL payment + migration](https://github.com/lightningnetwork/lnd/pull/10485) with + comprehensive tests. The migration is currently dev-only, compiled behind + the `test_db_postgres`, `test_db_sqlite`, or `test_native_sql` build tags. ## Code Health