diff --git a/.golangci.yml b/.golangci.yml index dc82da888..4cb893934 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -302,7 +302,9 @@ linters: - "internal\\/musig2v040" - channeldb/migration_01_to_11 - channeldb/migration/lnwire21 - + - payments/db/migration1/lnwire + - payments/db/migration1/record + issues: # Only show newly introduced problems. new-from-rev: 03eab4db64540aa5f789c617793e4459f4ba9e78 diff --git a/config_builder.go b/config_builder.go index 74f8b3afb..729cb6017 100644 --- a/config_builder.go +++ b/config_builder.go @@ -51,6 +51,8 @@ import ( "github.com/lightningnetwork/lnd/macaroons" "github.com/lightningnetwork/lnd/msgmux" paymentsdb "github.com/lightningnetwork/lnd/payments/db" + paymentsmig1 "github.com/lightningnetwork/lnd/payments/db/migration1" + paymentsmig1sqlc "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" "github.com/lightningnetwork/lnd/rpcperms" "github.com/lightningnetwork/lnd/signal" "github.com/lightningnetwork/lnd/sqldb" @@ -76,6 +78,10 @@ const ( // graphMigration is the version number for the graph migration // that migrates the KV graph to the native SQL schema. graphMigration = 10 + + // paymentMigration is the version number for the payments migration + // that migrates KV payments to the native SQL schema. + paymentMigration = 14 ) // GrpcRegistrar is an interface that must be satisfied by an external subserver @@ -1153,6 +1159,31 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( return nil } + paymentMig := func(tx *sqlc.Queries) error { + err := paymentsmig1.MigratePaymentsKVToSQL( + ctx, + dbs.ChanStateDB.Backend, + paymentsmig1sqlc.New(tx.GetTx()), + &paymentsmig1.SQLStoreConfig{ + QueryCfg: queryCfg, + }, + ) + if err != nil { + return fmt.Errorf("failed to migrate "+ + "payments to SQL: %w", err) + } + + // Set the payments bucket tombstone to + // indicate that the migration has been + // completed. + d.logger.Debugf("Setting payments bucket " + + "tombstone") + + return paymentsdb.SetPaymentsBucketTombstone( + dbs.ChanStateDB.Backend, + ) + } + // Make sure we attach the custom migration function to // the correct migration version. for i := 0; i < len(migrations); i++ { @@ -1162,11 +1193,17 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( migrations[i].MigrationFn = invoiceMig continue + case graphMigration: migrations[i].MigrationFn = graphMig continue + case paymentMigration: + migrations[i].MigrationFn = paymentMig + + continue + default: } @@ -1265,6 +1302,27 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( return nil, nil, err } + // Check if the payments bucket tombstone is set. If it is, we + // need to return and ask the user switch back to using the + // native SQL store. + ripPayments, err := paymentsdb.GetPaymentsBucketTombstone( + dbs.ChanStateDB.Backend, + ) + if err != nil { + err = fmt.Errorf("unable to check payments bucket "+ + "tombstone: %w", err) + d.logger.Error(err) + + return nil, nil, err + } + if ripPayments { + err = fmt.Errorf("payments bucket tombstoned, please " + + "switch back to native SQL") + d.logger.Error(err) + + return nil, nil, err + } + dbs.InvoiceDB = dbs.ChanStateDB graphStore, err = graphdb.NewKVStore( diff --git a/docs/release-notes/release-notes-0.21.0.md b/docs/release-notes/release-notes-0.21.0.md index 0679bb98b..5c456d2c4 100644 --- a/docs/release-notes/release-notes-0.21.0.md +++ b/docs/release-notes/release-notes-0.21.0.md @@ -217,6 +217,10 @@ db functions Part 2](https://github.com/lightningnetwork/lnd/pull/10308) * [Finalize SQL implementation for payments db](https://github.com/lightningnetwork/lnd/pull/10373) + * [Add the KV-to-SQL payment + migration](https://github.com/lightningnetwork/lnd/pull/10485) with + comprehensive tests. The migration is currently dev-only, compiled behind + the `test_db_postgres`, `test_db_sqlite`, or `test_native_sql` build tags. ## Code Health diff --git a/go.mod b/go.mod index 67c157080..7645bf62a 100644 --- a/go.mod +++ b/go.mod @@ -141,7 +141,7 @@ require ( github.com/opencontainers/runc v1.1.14 // indirect github.com/ory/dockertest/v3 v3.10.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect diff --git a/graph/db/migration1/test_sql.go b/graph/db/migration1/test_sql.go index 8034bec86..3d157dc9f 100644 --- a/graph/db/migration1/test_sql.go +++ b/graph/db/migration1/test_sql.go @@ -31,16 +31,15 @@ func (t *testBatchedSQLQueries) ExecTx(ctx context.Context, if err != nil { return err } - defer func() { - if err != nil { - _ = tx.Rollback() - } else { - err = tx.Commit() - } - }() reset() queries := sqlc.New(tx) - return txBody(queries) + if err := txBody(queries); err != nil { + _ = tx.Rollback() + + return err + } + + return tx.Commit() } diff --git a/payments/db/errors.go b/payments/db/errors.go index 0457db600..3b4e25bd9 100644 --- a/payments/db/errors.go +++ b/payments/db/errors.go @@ -136,6 +136,11 @@ var ( ErrNoDuplicateNestedBucket = errors.New("nested duplicate bucket not " + "found") + // ErrNoDuplicateSequenceNumber is returned when a duplicate payment + // sub-bucket does not contain the sequence number key. + ErrNoDuplicateSequenceNumber = errors.New("duplicate payment " + + "sequence number not found") + // ErrNoSequenceNrIndex is returned when an attempt to lookup a payment // index is made for a sequence number that is not indexed. // diff --git a/payments/db/kv_store.go b/payments/db/kv_store.go index 6d21048d7..b3fffe1e1 100644 --- a/payments/db/kv_store.go +++ b/payments/db/kv_store.go @@ -1236,7 +1236,7 @@ func fetchPaymentWithSequenceNumber(tx kvdb.RTx, paymentHash lntypes.Hash, seqBytes := subBucket.Get(duplicatePaymentSequenceKey) if seqBytes == nil { - return err + return ErrNoDuplicateSequenceNumber } // If this duplicate payment is not the sequence number we are diff --git a/payments/db/kv_tombstone.go b/payments/db/kv_tombstone.go new file mode 100644 index 000000000..7dc13b93b --- /dev/null +++ b/payments/db/kv_tombstone.go @@ -0,0 +1,71 @@ +package paymentsdb + +import ( + "fmt" + + "github.com/lightningnetwork/lnd/kvdb" +) + +var ( + // paymentsBucketTombstone is the key used to mark the payments bucket + // as permanently closed after a successful migration. + paymentsBucketTombstone = []byte("payments-tombstone") +) + +// SetPaymentsBucketTombstone sets the tombstone key in the payments bucket to +// mark the bucket as permanently closed. This prevents it from being reopened +// in the future. +func SetPaymentsBucketTombstone(db kvdb.Backend) error { + return kvdb.Update(db, func(tx kvdb.RwTx) error { + // Access the top-level payments bucket. + payments := tx.ReadWriteBucket(paymentsRootBucket) + + // In case the bucket doesn't exist, because we start + // immediately with the native SQL schema, we create it as well + // to make sure the user cannot switch back to the KV store. + if payments == nil { + var err error + payments, err = tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return fmt.Errorf("failed to create payments "+ + "bucket: %w", err) + } + } + + // Add the tombstone key to the payments bucket. + err := payments.Put(paymentsBucketTombstone, []byte("1")) + if err != nil { + return fmt.Errorf("failed to set tombstone: %w", err) + } + + return nil + }, func() {}) +} + +// GetPaymentsBucketTombstone checks if the tombstone key exists in the payments +// bucket. It returns true if the tombstone is present and false otherwise. +func GetPaymentsBucketTombstone(db kvdb.Backend) (bool, error) { + var tombstoneExists bool + + err := kvdb.View(db, func(tx kvdb.RTx) error { + // Access the top-level payments bucket. + payments := tx.ReadBucket(paymentsRootBucket) + if payments == nil { + tombstoneExists = false + return nil + } + + // Check if the tombstone key exists. + tombstone := payments.Get(paymentsBucketTombstone) + tombstoneExists = tombstone != nil + + return nil + }, func() {}) + if err != nil { + return false, err + } + + return tombstoneExists, nil +} diff --git a/payments/db/log.go b/payments/db/log.go index 8a77dbcec..c8892341d 100644 --- a/payments/db/log.go +++ b/payments/db/log.go @@ -3,6 +3,7 @@ package paymentsdb import ( "github.com/btcsuite/btclog/v2" "github.com/lightningnetwork/lnd/build" + paymentsmig1 "github.com/lightningnetwork/lnd/payments/db/migration1" ) // log is a logger that is initialized with no output filters. This @@ -29,4 +30,5 @@ func DisableLog() { // using btclog. func UseLogger(logger btclog.Logger) { log = logger + paymentsmig1.UseLogger(logger) } diff --git a/payments/db/migration1/codec.go b/payments/db/migration1/codec.go new file mode 100644 index 000000000..64225d170 --- /dev/null +++ b/payments/db/migration1/codec.go @@ -0,0 +1,198 @@ +package migration1 + +import ( + "encoding/binary" + "fmt" + "io" + "time" + + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" +) + +// Big endian is the preferred byte order, due to cursor scans over +// integer keys iterating in order. +var byteOrder = binary.BigEndian + +// UnknownElementType is an error returned when the codec is unable to encode +// or decode a particular type. +// +// NOTE: This is a frozen, self-contained copy of the serialization error type +// to ensure that this migration package has no dependency on live packages +// whose implementation may change after this migration was released. +type UnknownElementType struct { + method string + element interface{} +} + +// NewUnknownElementType creates a new UnknownElementType error from the passed +// method name and element. +func NewUnknownElementType(method string, el interface{}) UnknownElementType { + return UnknownElementType{method: method, element: el} +} + +// Error returns the name of the method that encountered the error, as well as +// the type that was unsupported. +func (e UnknownElementType) Error() string { + return fmt.Sprintf("Unknown type in %s: %T", e.method, e.element) +} + +// WriteElement serializes a single element into the provided io.Writer using +// big-endian byte order. Only the types required by the migration1 package are +// supported. This is a frozen, self-contained copy of the serialization logic +// to ensure that this migration package has no dependency on live packages +// whose implementation may change after this migration was released. +func WriteElement(w io.Writer, element interface{}) error { + switch e := element.(type) { + case paymentIndexType: + if err := binary.Write(w, byteOrder, e); err != nil { + return err + } + + case uint8: + if err := binary.Write(w, byteOrder, e); err != nil { + return err + } + + case uint32: + if err := binary.Write(w, byteOrder, e); err != nil { + return err + } + + case uint64: + if err := binary.Write(w, byteOrder, e); err != nil { + return err + } + + case lnwire.MilliSatoshi: + if err := binary.Write(w, byteOrder, uint64(e)); err != nil { + return err + } + + case [32]byte: + if _, err := w.Write(e[:]); err != nil { + return err + } + + case []byte: + if err := wire.WriteVarBytes(w, 0, e); err != nil { + return err + } + + default: + return UnknownElementType{"WriteElement", e} + } + + return nil +} + +// WriteElements serializes a variadic list of elements into the given +// io.Writer. +func WriteElements(w io.Writer, elements ...interface{}) error { + for _, element := range elements { + if err := WriteElement(w, element); err != nil { + return err + } + } + + return nil +} + +// ReadElement deserializes a single element from the provided io.Reader using +// big-endian byte order. Only the types required by the migration1 package are +// supported. This is a frozen, self-contained copy of the deserialization +// logic to ensure that this migration package has no dependency on live +// packages whose implementation may change after this migration was released. +func ReadElement(r io.Reader, element interface{}) error { + switch e := element.(type) { + case *paymentIndexType: + if err := binary.Read(r, byteOrder, e); err != nil { + return err + } + + case *uint8: + if err := binary.Read(r, byteOrder, e); err != nil { + return err + } + + case *uint32: + if err := binary.Read(r, byteOrder, e); err != nil { + return err + } + + case *uint64: + if err := binary.Read(r, byteOrder, e); err != nil { + return err + } + + case *lnwire.MilliSatoshi: + var a uint64 + if err := binary.Read(r, byteOrder, &a); err != nil { + return err + } + *e = lnwire.MilliSatoshi(a) + + case *[32]byte: + if _, err := io.ReadFull(r, e[:]); err != nil { + return err + } + + case *[]byte: + b, err := wire.ReadVarBytes(r, 0, 66000, "[]byte") + if err != nil { + return err + } + *e = b + + default: + return UnknownElementType{"ReadElement", e} + } + + return nil +} + +// ReadElements deserializes the provided io.Reader into a variadic list of +// target elements. +func ReadElements(r io.Reader, elements ...interface{}) error { + for _, element := range elements { + if err := ReadElement(r, element); err != nil { + return err + } + } + + return nil +} + +// deserializeTime deserializes time as unix nanoseconds. +func deserializeTime(r io.Reader) (time.Time, error) { + var scratch [8]byte + if _, err := io.ReadFull(r, scratch[:]); err != nil { + return time.Time{}, err + } + + // Convert to time.Time. Interpret unix nano time zero as a zero + // time.Time value. + unixNano := byteOrder.Uint64(scratch[:]) + if unixNano == 0 { + return time.Time{}, nil + } + + return time.Unix(0, int64(unixNano)), nil +} + +// serializeTime serializes time as unix nanoseconds. +func serializeTime(w io.Writer, t time.Time) error { + var scratch [8]byte + + // Convert to unix nano seconds, but only if time is non-zero. Calling + // UnixNano() on a zero time yields an undefined result. + var unixNano int64 + if !t.IsZero() { + unixNano = t.UnixNano() + } + + byteOrder.PutUint64(scratch[:], uint64(unixNano)) + _, err := w.Write(scratch[:]) + + return err +} diff --git a/payments/db/migration1/errors.go b/payments/db/migration1/errors.go new file mode 100644 index 000000000..7c2c98288 --- /dev/null +++ b/payments/db/migration1/errors.go @@ -0,0 +1,154 @@ +package migration1 + +import "errors" + +var ( + // ErrAlreadyPaid signals we have already paid this payment hash. + ErrAlreadyPaid = errors.New("invoice is already paid") + + // ErrPaymentInFlight signals that payment for this payment hash is + // already "in flight" on the network. + ErrPaymentInFlight = errors.New("payment is in transition") + + // ErrPaymentExists is returned when we try to initialize an already + // existing payment that is not failed. + ErrPaymentExists = errors.New("payment already exists") + + // ErrPaymentInternal is returned when performing the payment has a + // conflicting state, such as, + // - payment has StatusSucceeded but remaining amount is not zero. + // - payment has StatusInitiated but remaining amount is zero. + // - payment has StatusFailed but remaining amount is zero. + ErrPaymentInternal = errors.New("internal error") + + // ErrPaymentNotInitiated is returned if the payment wasn't initiated. + ErrPaymentNotInitiated = errors.New("payment isn't initiated") + + // ErrPaymentAlreadySucceeded is returned in the event we attempt to + // change the status of a payment already succeeded. + ErrPaymentAlreadySucceeded = errors.New("payment is already succeeded") + + // ErrPaymentAlreadyFailed is returned in the event we attempt to alter + // a failed payment. + ErrPaymentAlreadyFailed = errors.New("payment has already failed") + + // ErrUnknownPaymentStatus is returned when we do not recognize the + // existing state of a payment. + ErrUnknownPaymentStatus = errors.New("unknown payment status") + + // ErrPaymentTerminal is returned if we attempt to alter a payment that + // already has reached a terminal condition. + ErrPaymentTerminal = errors.New("payment has reached terminal " + + "condition") + + // ErrAttemptAlreadySettled is returned if we try to alter an already + // settled HTLC attempt. + ErrAttemptAlreadySettled = errors.New("attempt already settled") + + // ErrAttemptAlreadyFailed is returned if we try to alter an already + // failed HTLC attempt. + ErrAttemptAlreadyFailed = errors.New("attempt already failed") + + // ErrValueMismatch is returned if we try to register a non-MPP attempt + // with an amount that doesn't match the payment amount. + ErrValueMismatch = errors.New("attempted value doesn't match payment " + + "amount") + + // ErrValueExceedsAmt is returned if we try to register an attempt that + // would take the total sent amount above the payment amount. + ErrValueExceedsAmt = errors.New("attempted value exceeds payment " + + "amount") + + // ErrNonMPPayment is returned if we try to register an MPP attempt for + // a payment that already has a non-MPP attempt registered. + ErrNonMPPayment = errors.New("payment has non-MPP attempts") + + // ErrMPPayment is returned if we try to register a non-MPP attempt for + // a payment that already has an MPP attempt registered. + ErrMPPayment = errors.New("payment has MPP attempts") + + // ErrMPPRecordInBlindedPayment is returned if we try to register an + // attempt with an MPP record for a payment to a blinded path. + ErrMPPRecordInBlindedPayment = errors.New("blinded payment cannot " + + "contain MPP records") + + // ErrBlindedPaymentTotalAmountMismatch is returned if we try to + // register an HTLC shard to a blinded route where the total amount + // doesn't match existing shards. + ErrBlindedPaymentTotalAmountMismatch = errors.New("blinded path " + + "total amount mismatch") + + // ErrMixedBlindedAndNonBlindedPayments is returned if we try to + // register a non-blinded attempt to a payment which uses a blinded + // paths or vice versa. + ErrMixedBlindedAndNonBlindedPayments = errors.New("mixed blinded and " + + "non-blinded payments") + + // ErrBlindedPaymentMissingTotalAmount is returned if we try to + // register a blinded payment attempt where the final hop doesn't set + // the total amount. + ErrBlindedPaymentMissingTotalAmount = errors.New("blinded payment " + + "final hop must set total amount") + + // ErrMPPPaymentAddrMismatch is returned if we try to register an MPP + // shard where the payment address doesn't match existing shards. + ErrMPPPaymentAddrMismatch = errors.New("payment address mismatch") + + // ErrMPPTotalAmountMismatch is returned if we try to register an MPP + // shard where the total amount doesn't match existing shards. + ErrMPPTotalAmountMismatch = errors.New("mp payment total amount " + + "mismatch") + + // ErrPaymentPendingSettled is returned when we try to add a new + // attempt to a payment that has at least one of its HTLCs settled. + ErrPaymentPendingSettled = errors.New("payment has settled htlcs") + + // ErrPaymentPendingFailed is returned when we try to add a new attempt + // to a payment that already has a failure reason. + ErrPaymentPendingFailed = errors.New("payment has failure reason") + + // ErrSentExceedsTotal is returned if the payment's current total sent + // amount exceed the total amount. + ErrSentExceedsTotal = errors.New("total sent exceeds total amount") + + // ErrNoAttemptInfo is returned when no attempt info is stored yet. + ErrNoAttemptInfo = errors.New("unable to find attempt info for " + + "inflight payment") +) + +// KV backend specific errors. +var ( + // ErrNoSequenceNumber is returned if we look up a payment which does + // not have a sequence number. + ErrNoSequenceNumber = errors.New("sequence number not found") + + // ErrDuplicateNotFound is returned when we lookup a payment by its + // index and cannot find a payment with a matching sequence number. + ErrDuplicateNotFound = errors.New("duplicate payment not found") + + // ErrNoDuplicateBucket is returned when we expect to find duplicates + // when looking up a payment from its index, but the payment does not + // have any. + ErrNoDuplicateBucket = errors.New("expected duplicate bucket") + + // ErrNoDuplicateNestedBucket is returned if we do not find duplicate + // payments in their own sub-bucket. + ErrNoDuplicateNestedBucket = errors.New("nested duplicate bucket not " + + "found") + + // ErrNoDuplicateSequenceNumber is returned when a duplicate payment + // sub-bucket does not contain the sequence number key. + ErrNoDuplicateSequenceNumber = errors.New("duplicate payment " + + "sequence number not found") + + // ErrNoSequenceNrIndex is returned when an attempt to lookup a payment + // index is made for a sequence number that is not indexed. + // + // NOTE: Only used for the kv backend. + ErrNoSequenceNrIndex = errors.New("payment sequence number index " + + "does not exist") + + // errMaxPaymentsReached is used internally to signal that the maximum + // number of payments has been reached during a paginated query. + errMaxPaymentsReached = errors.New("max payments reached") +) diff --git a/payments/db/migration1/interface.go b/payments/db/migration1/interface.go new file mode 100644 index 000000000..7d47118e9 --- /dev/null +++ b/payments/db/migration1/interface.go @@ -0,0 +1,140 @@ +package migration1 + +import ( + "context" + + "github.com/lightningnetwork/lnd/lntypes" +) + +// DB represents the interface to the underlying payments database. +type DB interface { + PaymentReader + PaymentWriter +} + +// PaymentReader represents the interface to read operations from the payments +// database. +type PaymentReader interface { + // QueryPayments queries the payments database and should support + // pagination. + QueryPayments(ctx context.Context, query Query) (Response, error) + + // FetchPayment fetches the payment corresponding to the given payment + // hash. + FetchPayment(ctx context.Context, + paymentHash lntypes.Hash) (*MPPayment, error) + + // FetchInFlightPayments returns all payments with status InFlight. + FetchInFlightPayments(ctx context.Context) ([]*MPPayment, error) +} + +// PaymentWriter represents the interface to write operations to the payments +// database. +type PaymentWriter interface { + // DeletePayment deletes a payment from the DB given its payment hash. + DeletePayment(ctx context.Context, paymentHash lntypes.Hash, + failedAttemptsOnly bool) error + + // DeletePayments deletes all payments from the DB given the specified + // flags. + DeletePayments(ctx context.Context, failedOnly, + failedAttemptsOnly bool) (int, error) + + PaymentControl +} + +// PaymentControl represents the interface to control the payment lifecycle and +// its database operations. This interface represents the control flow of how +// a payment should be handled in the database. They are not just writing +// operations but they inherently represent the flow of a payment. The methods +// are called in the following order. +// +// 1. InitPayment. +// 2. RegisterAttempt (a payment can have multiple attempts). +// 3. SettleAttempt or FailAttempt (attempts can also fail as long as the +// sending amount will be eventually settled). +// 4. Payment succeeds or "Fail" is called. +// 5. DeleteFailedAttempts is called which will delete all failed attempts +// for a payment to clean up the database. +type PaymentControl interface { + // InitPayment checks that no other payment with the same payment hash + // exists in the database before creating a new payment. However, it + // should allow the user making a subsequent payment if the payment is + // in a Failed state. + InitPayment(context.Context, lntypes.Hash, *PaymentCreationInfo) error + + // RegisterAttempt atomically records the provided HTLCAttemptInfo. + // + // IMPORTANT: Callers MUST serialize calls to RegisterAttempt for the + // same payment hash. Concurrent calls will result in race conditions + // where both calls read the same initial payment state, validate + // against stale data, and could cause overpayment. For example: + // - Both goroutines fetch payment with 400 sats sent + // - Both validate sending 650 sats won't overpay (within limit) + // - Both commit successfully + // - Result: 1700 sats sent, exceeding the payment amount + // The payment router/controller layer is responsible for ensuring + // serialized access per payment hash. + RegisterAttempt(context.Context, lntypes.Hash, + *HTLCAttemptInfo) (*MPPayment, error) + + // SettleAttempt marks the given attempt settled with the preimage. If + // this is a multi shard payment, this might implicitly mean the + // full payment succeeded. + // + // After invoking this method, InitPayment should always return an + // error to prevent us from making duplicate payments to the same + // payment hash. The provided preimage is atomically saved to the DB + // for record keeping. + SettleAttempt(context.Context, lntypes.Hash, uint64, + *HTLCSettleInfo) (*MPPayment, error) + + // FailAttempt marks the given payment attempt failed. + FailAttempt(context.Context, lntypes.Hash, uint64, + *HTLCFailInfo) (*MPPayment, error) + + // Fail transitions a payment into the Failed state, and records + // the ultimate reason the payment failed. Note that this should only + // be called when all active attempts are already failed. After + // invoking this method, InitPayment should return nil on its next call + // for this payment hash, allowing the user to make a subsequent + // payment. + Fail(context.Context, lntypes.Hash, FailureReason) (*MPPayment, error) + + // DeleteFailedAttempts removes all failed HTLCs from the db. It should + // be called for a given payment whenever all inflight htlcs are + // completed, and the payment has reached a final terminal state. + DeleteFailedAttempts(context.Context, lntypes.Hash) error +} + +// DBMPPayment is an interface that represents the payment state during a +// payment lifecycle. +type DBMPPayment interface { + // GetState returns the current state of the payment. + GetState() *MPPaymentState + + // Terminated returns true if the payment is in a final state. + Terminated() bool + + // GetStatus returns the current status of the payment. + GetStatus() PaymentStatus + + // NeedWaitAttempts specifies whether the payment needs to wait for the + // outcome of an attempt. + NeedWaitAttempts() (bool, error) + + // GetHTLCs returns all HTLCs of this payment. + GetHTLCs() []HTLCAttempt + + // InFlightHTLCs returns all HTLCs that are in flight. + InFlightHTLCs() []HTLCAttempt + + // AllowMoreAttempts is used to decide whether we can safely attempt + // more HTLCs for a given payment state. Return an error if the payment + // is in an unexpected state. + AllowMoreAttempts() (bool, error) + + // TerminalInfo returns the settled HTLC attempt or the payment's + // failure reason. + TerminalInfo() (*HTLCAttempt, *FailureReason) +} diff --git a/payments/db/migration1/kv_duplicate_payments.go b/payments/db/migration1/kv_duplicate_payments.go new file mode 100644 index 000000000..1454430bc --- /dev/null +++ b/payments/db/migration1/kv_duplicate_payments.go @@ -0,0 +1,249 @@ +package migration1 + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" +) + +var ( + // duplicatePaymentsBucket is the name of a optional sub-bucket within + // the payment hash bucket, that is used to hold duplicate payments to a + // payment hash. This is needed to support information from earlier + // versions of lnd, where it was possible to pay to a payment hash more + // than once. + duplicatePaymentsBucket = []byte("payment-duplicate-bucket") + + // duplicatePaymentSettleInfoKey is a key used in the payment's + // sub-bucket to store the settle info of the payment. + duplicatePaymentSettleInfoKey = []byte("payment-settle-info") + + // duplicatePaymentAttemptInfoKey is a key used in the payment's + // sub-bucket to store the info about the latest attempt that was done + // for the payment in question. + duplicatePaymentAttemptInfoKey = []byte("payment-attempt-info") + + // duplicatePaymentCreationInfoKey is a key used in the payment's + // sub-bucket to store the creation info of the payment. + duplicatePaymentCreationInfoKey = []byte("payment-creation-info") + + // duplicatePaymentFailInfoKey is a key used in the payment's sub-bucket + // to store information about the reason a payment failed. + duplicatePaymentFailInfoKey = []byte("payment-fail-info") + + // duplicatePaymentSequenceKey is a key used in the payment's sub-bucket + // to store the sequence number of the payment. + duplicatePaymentSequenceKey = []byte("payment-sequence-key") +) + +// duplicateHTLCAttemptInfo contains static information about a specific HTLC +// attempt for a payment. This information is used by the router to handle any +// errors coming back after an attempt is made, and to query the switch about +// the status of the attempt. +type duplicateHTLCAttemptInfo struct { + // attemptID is the unique ID used for this attempt. + attemptID uint64 + + // sessionKey is the ephemeral key used for this attempt. + sessionKey [btcec.PrivKeyBytesLen]byte + + // route is the route attempted to send the HTLC. + route Route +} + +// fetchDuplicatePaymentStatus fetches the payment status of the payment. If +// the payment isn't found, it will return error `ErrPaymentNotInitiated`. +func fetchDuplicatePaymentStatus(bucket kvdb.RBucket) (PaymentStatus, error) { + if bucket.Get(duplicatePaymentSettleInfoKey) != nil { + return StatusSucceeded, nil + } + + if bucket.Get(duplicatePaymentFailInfoKey) != nil { + return StatusFailed, nil + } + + if bucket.Get(duplicatePaymentCreationInfoKey) != nil { + return StatusInFlight, nil + } + + return 0, ErrPaymentNotInitiated +} + +func deserializeDuplicateHTLCAttemptInfo(r io.Reader) ( + *duplicateHTLCAttemptInfo, error) { + + a := &duplicateHTLCAttemptInfo{} + err := ReadElements(r, &a.attemptID, &a.sessionKey) + if err != nil { + return nil, err + } + a.route, err = DeserializeRoute(r) + if err != nil { + return nil, err + } + + return a, nil +} + +func deserializeDuplicatePaymentCreationInfo(r io.Reader) ( + *PaymentCreationInfo, error) { + + var scratch [8]byte + + c := &PaymentCreationInfo{} + + if _, err := io.ReadFull(r, c.PaymentIdentifier[:]); err != nil { + return nil, err + } + + if _, err := io.ReadFull(r, scratch[:]); err != nil { + return nil, err + } + c.Value = lnwire.MilliSatoshi(byteOrder.Uint64(scratch[:])) + + if _, err := io.ReadFull(r, scratch[:]); err != nil { + return nil, err + } + c.CreationTime = time.Unix(int64(byteOrder.Uint64(scratch[:])), 0) + + if _, err := io.ReadFull(r, scratch[:4]); err != nil { + return nil, err + } + + reqLen := byteOrder.Uint32(scratch[:4]) + payReq := make([]byte, reqLen) + if reqLen > 0 { + if _, err := io.ReadFull(r, payReq); err != nil { + return nil, err + } + } + c.PaymentRequest = payReq + + return c, nil +} + +func fetchDuplicatePayment(bucket kvdb.RBucket) (*MPPayment, error) { + seqBytes := bucket.Get(duplicatePaymentSequenceKey) + if seqBytes == nil { + return nil, fmt.Errorf("sequence number not found") + } + + sequenceNum := binary.BigEndian.Uint64(seqBytes) + + // Get the payment status. + paymentStatus, err := fetchDuplicatePaymentStatus(bucket) + if err != nil { + return nil, err + } + + // Get the PaymentCreationInfo. + b := bucket.Get(duplicatePaymentCreationInfoKey) + if b == nil { + return nil, fmt.Errorf("creation info not found") + } + + r := bytes.NewReader(b) + creationInfo, err := deserializeDuplicatePaymentCreationInfo(r) + if err != nil { + return nil, err + } + + // Get failure reason if available. + var failureReason *FailureReason + b = bucket.Get(duplicatePaymentFailInfoKey) + if b != nil { + reason := FailureReason(b[0]) + failureReason = &reason + } + + payment := &MPPayment{ + SequenceNum: sequenceNum, + Info: creationInfo, + FailureReason: failureReason, + Status: paymentStatus, + } + + // Get the HTLCAttemptInfo. It can be absent. + b = bucket.Get(duplicatePaymentAttemptInfoKey) + if b != nil { + r = bytes.NewReader(b) + attempt, err := deserializeDuplicateHTLCAttemptInfo(r) + if err != nil { + return nil, err + } + + htlc := HTLCAttempt{ + HTLCAttemptInfo: HTLCAttemptInfo{ + AttemptID: attempt.attemptID, + Route: attempt.route, + sessionKey: attempt.sessionKey, + }, + } + + // Get the payment preimage. This is only found for + // successful payments. + b = bucket.Get(duplicatePaymentSettleInfoKey) + if b != nil { + var preimg lntypes.Preimage + copy(preimg[:], b) + + htlc.Settle = &HTLCSettleInfo{ + Preimage: preimg, + SettleTime: time.Time{}, + } + } else { + // Otherwise the payment must have failed. + htlc.Failure = &HTLCFailInfo{ + FailTime: time.Time{}, + } + } + + payment.HTLCs = []HTLCAttempt{htlc} + } + + return payment, nil +} + +func fetchDuplicatePayments(paymentHashBucket kvdb.RBucket) ([]*MPPayment, + error) { + + var payments []*MPPayment + + // For older versions of lnd, duplicate payments to a payment has was + // possible. These will be found in a sub-bucket indexed by their + // sequence number if available. + dup := paymentHashBucket.NestedReadBucket(duplicatePaymentsBucket) + if dup == nil { + return nil, nil + } + + err := dup.ForEach(func(k, v []byte) error { + subBucket := dup.NestedReadBucket(k) + if subBucket == nil { + // We one bucket for each duplicate to be found. + return fmt.Errorf("non bucket element" + + "in duplicate bucket") + } + + p, err := fetchDuplicatePayment(subBucket) + if err != nil { + return err + } + + payments = append(payments, p) + + return nil + }) + if err != nil { + return nil, err + } + + return payments, nil +} diff --git a/payments/db/migration1/kv_store.go b/payments/db/migration1/kv_store.go new file mode 100644 index 000000000..eda1945bc --- /dev/null +++ b/payments/db/migration1/kv_store.go @@ -0,0 +1,2120 @@ +package migration1 + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "sort" + "sync" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/record" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // paymentSeqBlockSize is the block size used when we batch allocate + // payment sequences for future payments. + paymentSeqBlockSize = 1000 + + // paymentProgressLogInterval is the interval we use limiting the + // logging output of payment processing. + paymentProgressLogInterval = 30 * time.Second +) + +//nolint:ll +var ( + // paymentsRootBucket is the name of the top-level bucket within the + // database that stores all data related to payments. Within this + // bucket, each payment hash its own sub-bucket keyed by its payment + // hash. + // + // Bucket hierarchy: + // + // root-bucket + // | + // |-- + // | |--sequence-key: + // | |--creation-info-key: + // | |--fail-info-key: <(optional) fail info> + // | | + // | |--payment-htlcs-bucket (shard-bucket) + // | | | + // | | |-- ai: + // | | |-- si: <(optional) settle info> + // | | |-- fi: <(optional) fail info> + // | | | + // | | ... + // | | + // | | + // | |--duplicate-bucket (only for old, completed payments) + // | | + // | |-- + // | | |--sequence-key: + // | | |--creation-info-key: + // | | |--ai: + // | | |--si: + // | | |--fi: + // | | + // | |-- + // | | | + // | ... ... + // | + // |-- + // | | + // | ... + // ... + // + paymentsRootBucket = []byte("payments-root-bucket") + + // paymentSequenceKey is a key used in the payment's sub-bucket to + // store the sequence number of the payment. + paymentSequenceKey = []byte("payment-sequence-key") + + // paymentCreationInfoKey is a key used in the payment's sub-bucket to + // store the creation info of the payment. + paymentCreationInfoKey = []byte("payment-creation-info") + + // paymentHtlcsBucket is a bucket where we'll store the information + // about the HTLCs that were attempted for a payment. + paymentHtlcsBucket = []byte("payment-htlcs-bucket") + + // htlcAttemptInfoKey is the key used as the prefix of an HTLC attempt + // to store the info about the attempt that was done for the HTLC in + // question. The HTLC attempt ID is concatenated at the end. + htlcAttemptInfoKey = []byte("ai") + + // htlcSettleInfoKey is the key used as the prefix of an HTLC attempt + // settle info, if any. The HTLC attempt ID is concatenated at the end. + htlcSettleInfoKey = []byte("si") + + // htlcFailInfoKey is the key used as the prefix of an HTLC attempt + // failure information, if any.The HTLC attempt ID is concatenated at + // the end. + htlcFailInfoKey = []byte("fi") + + // paymentFailInfoKey is a key used in the payment's sub-bucket to + // store information about the reason a payment failed. + paymentFailInfoKey = []byte("payment-fail-info") + + // paymentsIndexBucket is the name of the top-level bucket within the + // database that stores an index of payment sequence numbers to its + // payment hash. + // payments-sequence-index-bucket + // |--: + // |--... + // |--: + paymentsIndexBucket = []byte("payments-index-bucket") +) + +// KVStore implements persistence for payments and payment attempts. +type KVStore struct { + // Sequence management for the kv store. + seqMu sync.Mutex + currSeq uint64 + storedSeq uint64 + + // db is the underlying database implementation. + db kvdb.Backend +} + +// A compile-time constraint to ensure KVStore implements DB. +var _ DB = (*KVStore)(nil) + +// NewKVStore creates a new KVStore for payments. +func NewKVStore(db kvdb.Backend, + options ...OptionModifier) (*KVStore, error) { + + opts := DefaultOptions() + for _, applyOption := range options { + applyOption(opts) + } + + if !opts.NoMigration { + if err := initKVStore(db); err != nil { + return nil, err + } + } + + return &KVStore{ + db: db, + }, nil +} + +// paymentsTopLevelBuckets is a list of top-level buckets that are used for +// the payments database when using the kv store. +var paymentsTopLevelBuckets = [][]byte{ + paymentsRootBucket, + paymentsIndexBucket, +} + +// initKVStore creates and initializes the top-level buckets for the payment db. +func initKVStore(db kvdb.Backend) error { + err := kvdb.Update(db, func(tx kvdb.RwTx) error { + for _, tlb := range paymentsTopLevelBuckets { + if _, err := tx.CreateTopLevelBucket(tlb); err != nil { + return err + } + } + + return nil + }, func() {}) + if err != nil { + return fmt.Errorf("unable to create new payments db: %w", err) + } + + return nil +} + +// InitPayment checks or records the given PaymentCreationInfo with the DB, +// making sure it does not already exist as an in-flight payment. When this +// method returns successfully, the payment is guaranteed to be in the InFlight +// state. +func (p *KVStore) InitPayment(_ context.Context, paymentHash lntypes.Hash, + info *PaymentCreationInfo) error { + + // Obtain a new sequence number for this payment. This is used + // to sort the payments in order of creation, and also acts as + // a unique identifier for each payment. + sequenceNum, err := p.nextPaymentSequence() + if err != nil { + return err + } + + var b bytes.Buffer + if err := serializePaymentCreationInfo(&b, info); err != nil { + return err + } + infoBytes := b.Bytes() + + var updateErr error + err = kvdb.Batch(p.db, func(tx kvdb.RwTx) error { + // Reset the update error, to avoid carrying over an error + // from a previous execution of the batched db transaction. + updateErr = nil + + prefetchPayment(tx, paymentHash) + bucket, err := createPaymentBucket(tx, paymentHash) + if err != nil { + return err + } + + // Get the existing status of this payment, if any. + paymentStatus, err := fetchPaymentStatus(bucket) + + switch { + // If no error is returned, it means we already have this + // payment. We'll check the status to decide whether we allow + // retrying the payment or return a specific error. + case err == nil: + if err := paymentStatus.initializable(); err != nil { + updateErr = err + return nil + } + + // Otherwise, if the error is not `ErrPaymentNotInitiated`, + // we'll return the error. + case !errors.Is(err, ErrPaymentNotInitiated): + return err + } + + // Before we set our new sequence number, we check whether this + // payment has a previously set sequence number and remove its + // index entry if it exists. This happens in the case where we + // have a previously attempted payment which was left in a state + // where we can retry. + seqBytes := bucket.Get(paymentSequenceKey) + if seqBytes != nil { + indexBucket := tx.ReadWriteBucket(paymentsIndexBucket) + if err := indexBucket.Delete(seqBytes); err != nil { + return err + } + } + + // Once we have obtained a sequence number, we add an entry + // to our index bucket which will map the sequence number to + // our payment identifier. + err = createPaymentIndexEntry( + tx, sequenceNum, info.PaymentIdentifier, + ) + if err != nil { + return err + } + + err = bucket.Put(paymentSequenceKey, sequenceNum) + if err != nil { + return err + } + + // Add the payment info to the bucket, which contains the + // static information for this payment + err = bucket.Put(paymentCreationInfoKey, infoBytes) + if err != nil { + return err + } + + // We'll delete any lingering HTLCs to start with, in case we + // are initializing a payment that was attempted earlier, but + // left in a state where we could retry. + err = bucket.DeleteNestedBucket(paymentHtlcsBucket) + if err != nil && !errors.Is(err, kvdb.ErrBucketNotFound) { + return err + } + + // Also delete any lingering failure info now that we are + // re-attempting. + return bucket.Delete(paymentFailInfoKey) + }) + if err != nil { + return fmt.Errorf("unable to init payment: %w", err) + } + + return updateErr +} + +// DeleteFailedAttempts deletes all failed htlcs for a payment. +func (p *KVStore) DeleteFailedAttempts(ctx context.Context, + hash lntypes.Hash) error { + + const failedHtlcsOnly = true + err := p.DeletePayment(ctx, hash, failedHtlcsOnly) + if err != nil { + return err + } + + return nil +} + +// paymentIndexTypeHash is a payment index type which indicates that we have +// created an index of payment sequence number to payment hash. +type paymentIndexType uint8 + +// paymentIndexTypeHash is a payment index type which indicates that we have +// created an index of payment sequence number to payment hash. +const paymentIndexTypeHash paymentIndexType = 0 + +// createPaymentIndexEntry creates a payment hash typed index for a payment. The +// index produced contains a payment index type (which can be used in future to +// signal different payment index types) and the payment identifier. +func createPaymentIndexEntry(tx kvdb.RwTx, sequenceNumber []byte, + id lntypes.Hash) error { + + var b bytes.Buffer + if err := WriteElements(&b, paymentIndexTypeHash, id[:]); err != nil { + return err + } + + indexes := tx.ReadWriteBucket(paymentsIndexBucket) + + return indexes.Put(sequenceNumber, b.Bytes()) +} + +// deserializePaymentIndex deserializes a payment index entry. This function +// currently only supports deserialization of payment hash indexes, and will +// fail for other types. +func deserializePaymentIndex(r io.Reader) (lntypes.Hash, error) { + var ( + indexType paymentIndexType + paymentHash []byte + ) + + if err := ReadElements(r, &indexType, &paymentHash); err != nil { + return lntypes.Hash{}, err + } + + // While we only have on payment index type, we do not need to use our + // index type to deserialize the index. However, we sanity check that + // this type is as expected, since we had to read it out anyway. + if indexType != paymentIndexTypeHash { + return lntypes.Hash{}, fmt.Errorf("unknown payment index "+ + "type: %v", indexType) + } + + hash, err := lntypes.MakeHash(paymentHash) + if err != nil { + return lntypes.Hash{}, err + } + + return hash, nil +} + +// RegisterAttempt atomically records the provided HTLCAttemptInfo to the +// DB. +func (p *KVStore) RegisterAttempt(_ context.Context, paymentHash lntypes.Hash, + attempt *HTLCAttemptInfo) (*MPPayment, error) { + + // Serialize the information before opening the db transaction. + var a bytes.Buffer + err := serializeHTLCAttemptInfo(&a, attempt) + if err != nil { + return nil, err + } + htlcInfoBytes := a.Bytes() + + htlcIDBytes := make([]byte, 8) + binary.BigEndian.PutUint64(htlcIDBytes, attempt.AttemptID) + + var payment *MPPayment + err = kvdb.Batch(p.db, func(tx kvdb.RwTx) error { + prefetchPayment(tx, paymentHash) + bucket, err := fetchPaymentBucketUpdate(tx, paymentHash) + if err != nil { + return err + } + + payment, err = fetchPayment(bucket) + if err != nil { + return err + } + + // Check if registering a new attempt is allowed. + if err := payment.Registrable(); err != nil { + return err + } + + // Verify the attempt is compatible with the existing payment. + if err := verifyAttempt(payment, attempt); err != nil { + return err + } + + htlcsBucket, err := bucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + err = htlcsBucket.Put( + htlcBucketKey(htlcAttemptInfoKey, htlcIDBytes), + htlcInfoBytes, + ) + if err != nil { + return err + } + + // Retrieve attempt info for the notification. + payment, err = fetchPayment(bucket) + + return err + }) + if err != nil { + return nil, err + } + + return payment, err +} + +// SettleAttempt marks the given attempt settled with the preimage. If this is +// a multi shard payment, this might implicitly mean that the full payment +// succeeded. +// +// After invoking this method, InitPayment should always return an error to +// prevent us from making duplicate payments to the same payment hash. The +// provided preimage is atomically saved to the DB for record keeping. +func (p *KVStore) SettleAttempt(_ context.Context, hash lntypes.Hash, + attemptID uint64, settleInfo *HTLCSettleInfo) (*MPPayment, error) { + + var b bytes.Buffer + if err := serializeHTLCSettleInfo(&b, settleInfo); err != nil { + return nil, err + } + settleBytes := b.Bytes() + + return p.updateHtlcKey(hash, attemptID, htlcSettleInfoKey, settleBytes) +} + +// FailAttempt marks the given payment attempt failed. +func (p *KVStore) FailAttempt(_ context.Context, hash lntypes.Hash, + attemptID uint64, failInfo *HTLCFailInfo) (*MPPayment, error) { + + var b bytes.Buffer + if err := serializeHTLCFailInfo(&b, failInfo); err != nil { + return nil, err + } + failBytes := b.Bytes() + + return p.updateHtlcKey(hash, attemptID, htlcFailInfoKey, failBytes) +} + +// updateHtlcKey updates a database key for the specified htlc. +func (p *KVStore) updateHtlcKey(paymentHash lntypes.Hash, + attemptID uint64, key, value []byte) (*MPPayment, error) { + + aid := make([]byte, 8) + binary.BigEndian.PutUint64(aid, attemptID) + + var payment *MPPayment + err := kvdb.Batch(p.db, func(tx kvdb.RwTx) error { + payment = nil + + prefetchPayment(tx, paymentHash) + bucket, err := fetchPaymentBucketUpdate(tx, paymentHash) + if err != nil { + return err + } + + p, err := fetchPayment(bucket) + if err != nil { + return err + } + + // We can only update keys of in-flight payments. We allow + // updating keys even if the payment has reached a terminal + // condition, since the HTLC outcomes must still be updated. + if err := p.Status.updatable(); err != nil { + return err + } + + htlcsBucket := bucket.NestedReadWriteBucket(paymentHtlcsBucket) + if htlcsBucket == nil { + return fmt.Errorf("htlcs bucket not found") + } + + attemptKey := htlcBucketKey(htlcAttemptInfoKey, aid) + if htlcsBucket.Get(attemptKey) == nil { + return fmt.Errorf("HTLC with ID %v not registered", + attemptID) + } + + // Make sure the shard is not already failed or settled. + failKey := htlcBucketKey(htlcFailInfoKey, aid) + if htlcsBucket.Get(failKey) != nil { + return ErrAttemptAlreadyFailed + } + + settleKey := htlcBucketKey(htlcSettleInfoKey, aid) + if htlcsBucket.Get(settleKey) != nil { + return ErrAttemptAlreadySettled + } + + // Add or update the key for this htlc. + err = htlcsBucket.Put(htlcBucketKey(key, aid), value) + if err != nil { + return err + } + + // Retrieve attempt info for the notification. + payment, err = fetchPayment(bucket) + + return err + }) + if err != nil { + return nil, err + } + + return payment, err +} + +// Fail transitions a payment into the Failed state, and records the reason the +// payment failed. After invoking this method, InitPayment should return nil on +// its next call for this payment hash, allowing the switch to make a +// subsequent payment. +func (p *KVStore) Fail(_ context.Context, paymentHash lntypes.Hash, + reason FailureReason) (*MPPayment, error) { + + var ( + updateErr error + payment *MPPayment + ) + err := kvdb.Batch(p.db, func(tx kvdb.RwTx) error { + // Reset the update error, to avoid carrying over an error + // from a previous execution of the batched db transaction. + updateErr = nil + payment = nil + + prefetchPayment(tx, paymentHash) + bucket, err := fetchPaymentBucketUpdate(tx, paymentHash) + if errors.Is(err, ErrPaymentNotInitiated) { + updateErr = ErrPaymentNotInitiated + return nil + } else if err != nil { + return err + } + + // We mark the payment as failed as long as it is known. This + // lets the last attempt to fail with a terminal write its + // failure to the KVStore without synchronizing with + // other attempts. + _, err = fetchPaymentStatus(bucket) + if errors.Is(err, ErrPaymentNotInitiated) { + updateErr = ErrPaymentNotInitiated + return nil + } else if err != nil { + return err + } + + // Put the failure reason in the bucket for record keeping. + v := []byte{byte(reason)} + err = bucket.Put(paymentFailInfoKey, v) + if err != nil { + return err + } + + // Retrieve attempt info for the notification, if available. + payment, err = fetchPayment(bucket) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return nil, err + } + + return payment, updateErr +} + +// FetchPayment returns information about a payment from the database. +func (p *KVStore) FetchPayment(_ context.Context, + paymentHash lntypes.Hash) (*MPPayment, error) { + + var payment *MPPayment + err := kvdb.View(p.db, func(tx kvdb.RTx) error { + prefetchPayment(tx, paymentHash) + bucket, err := fetchPaymentBucket(tx, paymentHash) + if err != nil { + return err + } + + payment, err = fetchPayment(bucket) + + return err + }, func() { + payment = nil + }) + if err != nil { + return nil, err + } + + return payment, nil +} + +// prefetchPayment attempts to prefetch as much of the payment as possible to +// reduce DB roundtrips. +func prefetchPayment(tx kvdb.RTx, paymentHash lntypes.Hash) { + rb := kvdb.RootBucket(tx) + kvdb.Prefetch( + rb, + []string{ + // Prefetch all keys in the payment's bucket. + string(paymentsRootBucket), + string(paymentHash[:]), + }, + []string{ + // Prefetch all keys in the payment's htlc bucket. + string(paymentsRootBucket), + string(paymentHash[:]), + string(paymentHtlcsBucket), + }, + ) +} + +// createPaymentBucket creates or fetches the sub-bucket assigned to this +// payment hash. +func createPaymentBucket(tx kvdb.RwTx, paymentHash lntypes.Hash) ( + kvdb.RwBucket, error) { + + payments, err := tx.CreateTopLevelBucket(paymentsRootBucket) + if err != nil { + return nil, err + } + + return payments.CreateBucketIfNotExists(paymentHash[:]) +} + +// fetchPaymentBucket fetches the sub-bucket assigned to this payment hash. If +// the bucket does not exist, it returns ErrPaymentNotInitiated. +func fetchPaymentBucket(tx kvdb.RTx, paymentHash lntypes.Hash) ( + kvdb.RBucket, error) { + + payments := tx.ReadBucket(paymentsRootBucket) + if payments == nil { + return nil, ErrPaymentNotInitiated + } + + bucket := payments.NestedReadBucket(paymentHash[:]) + if bucket == nil { + return nil, ErrPaymentNotInitiated + } + + return bucket, nil +} + +// fetchPaymentBucketUpdate is identical to fetchPaymentBucket, but it returns a +// bucket that can be written to. +func fetchPaymentBucketUpdate(tx kvdb.RwTx, paymentHash lntypes.Hash) ( + kvdb.RwBucket, error) { + + payments := tx.ReadWriteBucket(paymentsRootBucket) + if payments == nil { + return nil, ErrPaymentNotInitiated + } + + bucket := payments.NestedReadWriteBucket(paymentHash[:]) + if bucket == nil { + return nil, ErrPaymentNotInitiated + } + + return bucket, nil +} + +// nextPaymentSequence returns the next sequence number to store for a new +// payment. +func (p *KVStore) nextPaymentSequence() ([]byte, error) { + p.seqMu.Lock() + defer p.seqMu.Unlock() + + // Set a new upper bound in the DB every 1000 payments to avoid + // conflicts on the sequence when using etcd. + if p.currSeq == p.storedSeq { + var currPaymentSeq, newUpperBound uint64 + if err := kvdb.Update(p.db, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + currPaymentSeq = paymentsBucket.Sequence() + newUpperBound = currPaymentSeq + paymentSeqBlockSize + + return paymentsBucket.SetSequence(newUpperBound) + }, func() {}); err != nil { + return nil, err + } + + // We lazy initialize the cached currPaymentSeq here using the + // first nextPaymentSequence() call. This if statement will auto + // initialize our stored currPaymentSeq, since by default both + // this variable and storedPaymentSeq are zero which in turn + // will have us fetch the current values from the DB. + if p.currSeq == 0 { + p.currSeq = currPaymentSeq + } + + p.storedSeq = newUpperBound + } + + p.currSeq++ + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, p.currSeq) + + return b, nil +} + +// fetchPaymentStatus fetches the payment status of the payment. If the payment +// isn't found, it will return error `ErrPaymentNotInitiated`. +func fetchPaymentStatus(bucket kvdb.RBucket) (PaymentStatus, error) { + // Creation info should be set for all payments, regardless of state. + // If not, it is unknown. + if bucket.Get(paymentCreationInfoKey) == nil { + return 0, ErrPaymentNotInitiated + } + + payment, err := fetchPayment(bucket) + if err != nil { + return 0, err + } + + return payment.Status, nil +} + +// FetchInFlightPayments returns all payments with status InFlight. +func (p *KVStore) FetchInFlightPayments(_ context.Context) ([]*MPPayment, + error) { + + var ( + inFlights []*MPPayment + start = time.Now() + lastLogTime = time.Now() + processedCount int + ) + + err := kvdb.View(p.db, func(tx kvdb.RTx) error { + payments := tx.ReadBucket(paymentsRootBucket) + if payments == nil { + return nil + } + + return payments.ForEach(func(k, _ []byte) error { + bucket := payments.NestedReadBucket(k) + if bucket == nil { + return fmt.Errorf("non bucket element") + } + + p, err := fetchPayment(bucket) + if err != nil { + return err + } + + processedCount++ + if time.Since(lastLogTime) >= + paymentProgressLogInterval { + + log.Debugf("Scanning inflight payments "+ + "(in progress), processed %d, last "+ + "processed payment: %v", processedCount, + p.Info) + + lastLogTime = time.Now() + } + + // Skip the payment if it's terminated. + if p.Terminated() { + return nil + } + + inFlights = append(inFlights, p) + + return nil + }) + }, func() { + inFlights = nil + }) + if err != nil { + return nil, err + } + + elapsed := time.Since(start) + log.Debugf("Completed scanning for inflight payments: "+ + "total_processed=%d, found_inflight=%d, elapsed=%v", + processedCount, len(inFlights), + elapsed.Round(time.Millisecond)) + + return inFlights, nil +} + +// htlcBucketKey creates a composite key from prefix and id where the result is +// simply the two concatenated. +func htlcBucketKey(prefix, id []byte) []byte { + key := make([]byte, len(prefix)+len(id)) + copy(key, prefix) + copy(key[len(prefix):], id) + + return key +} + +// FetchPayments returns all sent payments found in the DB. +func (p *KVStore) FetchPayments() ([]*MPPayment, error) { + var payments []*MPPayment + + err := kvdb.View(p.db, func(tx kvdb.RTx) error { + paymentsBucket := tx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + return nil + } + + return paymentsBucket.ForEach(func(k, v []byte) error { + bucket := paymentsBucket.NestedReadBucket(k) + if bucket == nil { + // We only expect sub-buckets to be found in + // this top-level bucket. + return fmt.Errorf("non bucket element in " + + "payments bucket") + } + + p, err := fetchPayment(bucket) + if err != nil { + return err + } + + payments = append(payments, p) + + // For older versions of lnd, duplicate payments to a + // payment has was possible. These will be found in a + // sub-bucket indexed by their sequence number if + // available. + duplicatePayments, err := fetchDuplicatePayments(bucket) + if err != nil { + return err + } + + payments = append(payments, duplicatePayments...) + + return nil + }) + }, func() { + payments = nil + }) + if err != nil { + return nil, err + } + + // Before returning, sort the payments by their sequence number. + sort.Slice(payments, func(i, j int) bool { + return payments[i].SequenceNum < payments[j].SequenceNum + }) + + return payments, nil +} + +func fetchCreationInfo(bucket kvdb.RBucket) (*PaymentCreationInfo, error) { + b := bucket.Get(paymentCreationInfoKey) + if b == nil { + return nil, fmt.Errorf("creation info not found") + } + + r := bytes.NewReader(b) + + return deserializePaymentCreationInfo(r) +} + +func fetchPayment(bucket kvdb.RBucket) (*MPPayment, error) { + seqBytes := bucket.Get(paymentSequenceKey) + if seqBytes == nil { + return nil, fmt.Errorf("sequence number not found") + } + + sequenceNum := binary.BigEndian.Uint64(seqBytes) + + // Get the PaymentCreationInfo. + creationInfo, err := fetchCreationInfo(bucket) + if err != nil { + return nil, err + } + + var htlcs []HTLCAttempt + htlcsBucket := bucket.NestedReadBucket(paymentHtlcsBucket) + if htlcsBucket != nil { + // Get the payment attempts. This can be empty. + htlcs, err = fetchHtlcAttempts(htlcsBucket) + if err != nil { + return nil, err + } + } + + // Get failure reason if available. + var failureReason *FailureReason + b := bucket.Get(paymentFailInfoKey) + if b != nil { + reason := FailureReason(b[0]) + failureReason = &reason + } + + // Create a new payment. + payment := &MPPayment{ + SequenceNum: sequenceNum, + Info: creationInfo, + HTLCs: htlcs, + FailureReason: failureReason, + } + + // Set its state and status. + if err := payment.setState(); err != nil { + return nil, err + } + + return payment, nil +} + +// fetchHtlcAttempts retrieves all htlc attempts made for the payment found in +// the given bucket. +func fetchHtlcAttempts(bucket kvdb.RBucket) ([]HTLCAttempt, error) { + htlcsMap := make(map[uint64]*HTLCAttempt) + + attemptInfoCount := 0 + err := bucket.ForEach(func(k, v []byte) error { + aid := byteOrder.Uint64(k[len(k)-8:]) + + if _, ok := htlcsMap[aid]; !ok { + htlcsMap[aid] = &HTLCAttempt{} + } + + var err error + switch { + case bytes.HasPrefix(k, htlcAttemptInfoKey): + attemptInfo, err := readHtlcAttemptInfo(v) + if err != nil { + return err + } + + attemptInfo.AttemptID = aid + htlcsMap[aid].HTLCAttemptInfo = *attemptInfo + attemptInfoCount++ + + case bytes.HasPrefix(k, htlcSettleInfoKey): + htlcsMap[aid].Settle, err = readHtlcSettleInfo(v) + if err != nil { + return err + } + + case bytes.HasPrefix(k, htlcFailInfoKey): + htlcsMap[aid].Failure, err = readHtlcFailInfo(v) + if err != nil { + return err + } + + default: + return fmt.Errorf("unknown htlc attempt key") + } + + return nil + }) + if err != nil { + return nil, err + } + + // Sanity check that all htlcs have an attempt info. + if attemptInfoCount != len(htlcsMap) { + return nil, ErrNoAttemptInfo + } + + keys := make([]uint64, len(htlcsMap)) + i := 0 + for k := range htlcsMap { + keys[i] = k + i++ + } + + // Sort HTLC attempts by their attempt ID. This is needed because in the + // DB we store the attempts with keys prefixed by their status which + // changes order (groups them together by status). + sort.Slice(keys, func(i, j int) bool { + return keys[i] < keys[j] + }) + + htlcs := make([]HTLCAttempt, len(htlcsMap)) + for i, key := range keys { + htlcs[i] = *htlcsMap[key] + } + + return htlcs, nil +} + +// readHtlcAttemptInfo reads the payment attempt info for this htlc. +func readHtlcAttemptInfo(b []byte) (*HTLCAttemptInfo, error) { + r := bytes.NewReader(b) + return deserializeHTLCAttemptInfo(r) +} + +// readHtlcSettleInfo reads the settle info for the htlc. If the htlc isn't +// settled, nil is returned. +func readHtlcSettleInfo(b []byte) (*HTLCSettleInfo, error) { + r := bytes.NewReader(b) + return deserializeHTLCSettleInfo(r) +} + +// readHtlcFailInfo reads the failure info for the htlc. If the htlc hasn't +// failed, nil is returned. +func readHtlcFailInfo(b []byte) (*HTLCFailInfo, error) { + r := bytes.NewReader(b) + return deserializeHTLCFailInfo(r) +} + +// fetchFailedHtlcKeys retrieves the bucket keys of all failed HTLCs of a +// payment bucket. +func fetchFailedHtlcKeys(bucket kvdb.RBucket) ([][]byte, error) { + htlcsBucket := bucket.NestedReadBucket(paymentHtlcsBucket) + + var htlcs []HTLCAttempt + var err error + if htlcsBucket != nil { + htlcs, err = fetchHtlcAttempts(htlcsBucket) + if err != nil { + return nil, err + } + } + + // Now iterate though them and save the bucket keys for the failed + // HTLCs. + var htlcKeys [][]byte + for _, h := range htlcs { + if h.Failure == nil { + continue + } + + htlcKeyBytes := make([]byte, 8) + binary.BigEndian.PutUint64(htlcKeyBytes, h.AttemptID) + + htlcKeys = append(htlcKeys, htlcKeyBytes) + } + + return htlcKeys, nil +} + +// QueryPayments is a query to the payments database which is restricted +// to a subset of payments by the payments query, containing an offset +// index and a maximum number of returned payments. +func (p *KVStore) QueryPayments(_ context.Context, + query Query) (Response, error) { + + var resp Response + + if err := kvdb.View(p.db, func(tx kvdb.RTx) error { + // Get the root payments bucket. + paymentsBucket := tx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + return nil + } + + // Get the index bucket which maps sequence number -> payment + // hash and duplicate bool. If we have a payments bucket, we + // should have an indexes bucket as well. + indexes := tx.ReadBucket(paymentsIndexBucket) + if indexes == nil { + return fmt.Errorf("index bucket does not exist") + } + + // accumulatePayments gets payments with the sequence number + // and hash provided and adds them to our list of payments if + // they meet the criteria of our query. It returns the number + // of payments that were added. + accumulatePayments := func(sequenceKey, hash []byte) (bool, + error) { + + r := bytes.NewReader(hash) + paymentHash, err := deserializePaymentIndex(r) + if err != nil { + return false, err + } + + payment, err := fetchPaymentWithSequenceNumber( + tx, paymentHash, sequenceKey, + ) + if err != nil { + return false, err + } + + // To keep compatibility with the old API, we only + // return non-succeeded payments if requested. + if payment.Status != StatusSucceeded && + !query.IncludeIncomplete { + + return false, err + } + + // Get the creation time in Unix seconds, this always + // rounds down the nanoseconds to full seconds. + createTime := payment.Info.CreationTime.Unix() + + // Skip any payments that were created before the + // specified time. + if createTime < query.CreationDateStart { + return false, nil + } + + // Skip any payments that were created after the + // specified time. + if query.CreationDateEnd != 0 && + createTime > query.CreationDateEnd { + + return false, nil + } + + // At this point, we've exhausted the offset, so we'll + // begin collecting invoices found within the range. + resp.Payments = append(resp.Payments, payment) + + return true, nil + } + + // Create a paginator which reads from our sequence index bucket + // with the parameters provided by the payments query. + paginator := channeldb.NewPaginator( + indexes.ReadCursor(), query.Reversed, query.IndexOffset, + query.MaxPayments, + ) + + // Run a paginated query, adding payments to our response. + if err := paginator.Query(accumulatePayments); err != nil { + return err + } + + // Counting the total number of payments is expensive, since we + // literally have to traverse the cursor linearly, which can + // take quite a while. So it's an optional query parameter. + if query.CountTotal { + var ( + totalPayments uint64 + err error + ) + countFn := func(_, _ []byte) error { + totalPayments++ + + return nil + } + + // In non-boltdb database backends, there's a faster + // ForAll query that allows for batch fetching items. + fastBucket, ok := indexes.(kvdb.ExtendedRBucket) + if ok { + err = fastBucket.ForAll(countFn) + } else { + err = indexes.ForEach(countFn) + } + if err != nil { + return fmt.Errorf("error counting payments: %w", + err) + } + + resp.TotalCount = totalPayments + } + + return nil + }, func() { + resp = Response{} + }); err != nil { + return resp, err + } + + // Need to swap the payments slice order if reversed order. + if query.Reversed { + for l, r := 0, len(resp.Payments)-1; l < r; l, r = l+1, r-1 { + resp.Payments[l], resp.Payments[r] = + resp.Payments[r], resp.Payments[l] + } + } + + // Set the first and last index of the returned payments so that the + // caller can resume from this point later on. + if len(resp.Payments) > 0 { + resp.FirstIndexOffset = resp.Payments[0].SequenceNum + resp.LastIndexOffset = + resp.Payments[len(resp.Payments)-1].SequenceNum + } + + return resp, nil +} + +// fetchPaymentWithSequenceNumber get the payment which matches the payment hash +// *and* sequence number provided from the database. This is required because +// we previously had more than one payment per hash, so we have multiple indexes +// pointing to a single payment; we want to retrieve the correct one. +func fetchPaymentWithSequenceNumber(tx kvdb.RTx, paymentHash lntypes.Hash, + sequenceNumber []byte) (*MPPayment, error) { + + // We can now lookup the payment keyed by its hash in + // the payments root bucket. + bucket, err := fetchPaymentBucket(tx, paymentHash) + if err != nil { + return nil, err + } + + // A single payment hash can have multiple payments associated with it. + // We lookup our sequence number first, to determine whether this is + // the payment we are actually looking for. + seqBytes := bucket.Get(paymentSequenceKey) + if seqBytes == nil { + return nil, ErrNoSequenceNumber + } + + // If this top level payment has the sequence number we are looking for, + // return it. + if bytes.Equal(seqBytes, sequenceNumber) { + return fetchPayment(bucket) + } + + // If we were not looking for the top level payment, we are looking for + // one of our duplicate payments. We need to iterate through the seq + // numbers in this bucket to find the correct payments. If we do not + // find a duplicate payments bucket here, something is wrong. + dup := bucket.NestedReadBucket(duplicatePaymentsBucket) + if dup == nil { + return nil, ErrNoDuplicateBucket + } + + var duplicatePayment *MPPayment + err = dup.ForEach(func(k, v []byte) error { + subBucket := dup.NestedReadBucket(k) + if subBucket == nil { + // We one bucket for each duplicate to be found. + return ErrNoDuplicateNestedBucket + } + + seqBytes := subBucket.Get(duplicatePaymentSequenceKey) + if seqBytes == nil { + return ErrNoDuplicateSequenceNumber + } + + // If this duplicate payment is not the sequence number we are + // looking for, we can continue. + if !bytes.Equal(seqBytes, sequenceNumber) { + return nil + } + + duplicatePayment, err = fetchDuplicatePayment(subBucket) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return nil, err + } + + // If none of the duplicate payments matched our sequence number, we + // failed to find the payment with this sequence number; something is + // wrong. + if duplicatePayment == nil { + return nil, ErrDuplicateNotFound + } + + return duplicatePayment, nil +} + +// DeletePayment deletes a payment from the DB given its payment hash. If +// failedHtlcsOnly is set, only failed HTLC attempts of the payment will be +// deleted. +func (p *KVStore) DeletePayment(_ context.Context, paymentHash lntypes.Hash, + failedHtlcsOnly bool) error { + + return kvdb.Update(p.db, func(tx kvdb.RwTx) error { + payments := tx.ReadWriteBucket(paymentsRootBucket) + if payments == nil { + return nil + } + + bucket := payments.NestedReadWriteBucket(paymentHash[:]) + if bucket == nil { + return fmt.Errorf("non bucket element in payments " + + "bucket") + } + + // If the status is InFlight, we cannot safely delete + // the payment information, so we return early. + paymentStatus, err := fetchPaymentStatus(bucket) + if err != nil { + return err + } + + // If the payment has inflight HTLCs, we cannot safely delete + // the payment information, so we return an error. + if err := paymentStatus.removable(); err != nil { + return fmt.Errorf("payment '%v' has inflight HTLCs"+ + "and therefore cannot be deleted: %w", + paymentHash.String(), err) + } + + // Delete the failed HTLC attempts we found. + if failedHtlcsOnly { + toDelete, err := fetchFailedHtlcKeys(bucket) + if err != nil { + return err + } + + htlcsBucket := bucket.NestedReadWriteBucket( + paymentHtlcsBucket, + ) + + for _, htlcID := range toDelete { + err = htlcsBucket.Delete( + htlcBucketKey( + htlcAttemptInfoKey, htlcID, + ), + ) + if err != nil { + return err + } + + err = htlcsBucket.Delete( + htlcBucketKey(htlcFailInfoKey, htlcID), + ) + if err != nil { + return err + } + + err = htlcsBucket.Delete( + htlcBucketKey( + htlcSettleInfoKey, htlcID, + ), + ) + if err != nil { + return err + } + } + + return nil + } + + seqNrs, err := fetchSequenceNumbers(bucket) + if err != nil { + return err + } + + err = payments.DeleteNestedBucket(paymentHash[:]) + if err != nil { + return err + } + + indexBucket := tx.ReadWriteBucket(paymentsIndexBucket) + for _, k := range seqNrs { + if err := indexBucket.Delete(k); err != nil { + return err + } + } + + return nil + }, func() {}) +} + +// DeletePayments deletes all completed and failed payments from the DB. If +// failedOnly is set, only failed payments will be considered for deletion. If +// failedHtlcsOnly is set, the payment itself won't be deleted, only failed HTLC +// attempts. The method returns the number of deleted payments, which is always +// 0 if failedHtlcsOnly is set. +func (p *KVStore) DeletePayments(_ context.Context, failedOnly, + failedHtlcsOnly bool) (int, error) { + + var numPayments int + err := kvdb.Update(p.db, func(tx kvdb.RwTx) error { + payments := tx.ReadWriteBucket(paymentsRootBucket) + if payments == nil { + return nil + } + + var ( + // deleteBuckets is the set of payment buckets we need + // to delete. + deleteBuckets [][]byte + + // deleteIndexes is the set of indexes pointing to these + // payments that need to be deleted. + deleteIndexes [][]byte + + // deleteHtlcs maps a payment hash to the HTLC IDs we + // want to delete for that payment. + deleteHtlcs = make(map[lntypes.Hash][][]byte) + ) + err := payments.ForEach(func(k, _ []byte) error { + bucket := payments.NestedReadBucket(k) + if bucket == nil { + // We only expect sub-buckets to be found in + // this top-level bucket. + return fmt.Errorf("non bucket element in " + + "payments bucket") + } + + // If the status is InFlight, we cannot safely delete + // the payment information, so we return early. + paymentStatus, err := fetchPaymentStatus(bucket) + if err != nil { + return err + } + + // If the payment has inflight HTLCs, we cannot safely + // delete the payment information, so we return an nil + // to skip it. + if err := paymentStatus.removable(); err != nil { + return nil + } + + // If we requested to only delete failed payments, we + // can return if this one is not. + if failedOnly && paymentStatus != StatusFailed { + return nil + } + + // If we are only deleting failed HTLCs, fetch them. + if failedHtlcsOnly { + toDelete, err := fetchFailedHtlcKeys(bucket) + if err != nil { + return err + } + + hash, err := lntypes.MakeHash(k) + if err != nil { + return err + } + + deleteHtlcs[hash] = toDelete + + // We return, we are only deleting attempts. + return nil + } + + // Add the bucket to the set of buckets we can delete. + deleteBuckets = append(deleteBuckets, k) + + // Get all the sequence number associated with the + // payment, including duplicates. + seqNrs, err := fetchSequenceNumbers(bucket) + if err != nil { + return err + } + + deleteIndexes = append(deleteIndexes, seqNrs...) + numPayments++ + + return nil + }) + if err != nil { + return err + } + + // Delete the failed HTLC attempts we found. + for hash, htlcIDs := range deleteHtlcs { + bucket := payments.NestedReadWriteBucket(hash[:]) + htlcsBucket := bucket.NestedReadWriteBucket( + paymentHtlcsBucket, + ) + + for _, aid := range htlcIDs { + if err := htlcsBucket.Delete( + htlcBucketKey(htlcAttemptInfoKey, aid), + ); err != nil { + return err + } + + if err := htlcsBucket.Delete( + htlcBucketKey(htlcFailInfoKey, aid), + ); err != nil { + return err + } + + if err := htlcsBucket.Delete( + htlcBucketKey(htlcSettleInfoKey, aid), + ); err != nil { + return err + } + } + } + + for _, k := range deleteBuckets { + if err := payments.DeleteNestedBucket(k); err != nil { + return err + } + } + + // Get our index bucket and delete all indexes pointing to the + // payments we are deleting. + indexBucket := tx.ReadWriteBucket(paymentsIndexBucket) + for _, k := range deleteIndexes { + if err := indexBucket.Delete(k); err != nil { + return err + } + } + + return nil + }, func() { + numPayments = 0 + }) + if err != nil { + return 0, err + } + + return numPayments, nil +} + +// fetchSequenceNumbers fetches all the sequence numbers associated with a +// payment, including those belonging to any duplicate payments. +func fetchSequenceNumbers(paymentBucket kvdb.RBucket) ([][]byte, error) { + seqNum := paymentBucket.Get(paymentSequenceKey) + if seqNum == nil { + return nil, errors.New("expected sequence number") + } + + sequenceNumbers := [][]byte{seqNum} + + // Get the duplicate payments bucket, if it has no duplicates, just + // return early with the payment sequence number. + duplicates := paymentBucket.NestedReadBucket(duplicatePaymentsBucket) + if duplicates == nil { + return sequenceNumbers, nil + } + + // If we do have duplicated, they are keyed by sequence number, so we + // iterate through the duplicates bucket and add them to our set of + // sequence numbers. + if err := duplicates.ForEach(func(k, v []byte) error { + sequenceNumbers = append(sequenceNumbers, k) + return nil + }); err != nil { + return nil, err + } + + return sequenceNumbers, nil +} + +func serializePaymentCreationInfo(w io.Writer, c *PaymentCreationInfo) error { + var scratch [8]byte + + if _, err := w.Write(c.PaymentIdentifier[:]); err != nil { + return err + } + + byteOrder.PutUint64(scratch[:], uint64(c.Value)) + if _, err := w.Write(scratch[:]); err != nil { + return err + } + + if err := serializeTime(w, c.CreationTime); err != nil { + return err + } + + byteOrder.PutUint32(scratch[:4], uint32(len(c.PaymentRequest))) + if _, err := w.Write(scratch[:4]); err != nil { + return err + } + + if _, err := w.Write(c.PaymentRequest); err != nil { + return err + } + + // Any remaining bytes are TLV encoded records. Currently, these are + // only the custom records provided by the user to be sent to the first + // hop. But this can easily be extended with further records by merging + // the records into a single TLV stream. + err := c.FirstHopCustomRecords.SerializeTo(w) + if err != nil { + return err + } + + return nil +} + +func deserializePaymentCreationInfo(r io.Reader) (*PaymentCreationInfo, + error) { + + var scratch [8]byte + + c := &PaymentCreationInfo{} + + if _, err := io.ReadFull(r, c.PaymentIdentifier[:]); err != nil { + return nil, err + } + + if _, err := io.ReadFull(r, scratch[:]); err != nil { + return nil, err + } + c.Value = lnwire.MilliSatoshi(byteOrder.Uint64(scratch[:])) + + creationTime, err := deserializeTime(r) + if err != nil { + return nil, err + } + c.CreationTime = creationTime + + if _, err := io.ReadFull(r, scratch[:4]); err != nil { + return nil, err + } + + reqLen := byteOrder.Uint32(scratch[:4]) + payReq := make([]byte, reqLen) + if reqLen > 0 { + if _, err := io.ReadFull(r, payReq); err != nil { + return nil, err + } + } + c.PaymentRequest = payReq + + // Any remaining bytes are TLV encoded records. Currently, these are + // only the custom records provided by the user to be sent to the first + // hop. But this can easily be extended with further records by merging + // the records into a single TLV stream. + c.FirstHopCustomRecords, err = lnwire.ParseCustomRecordsFrom(r) + if err != nil { + return nil, err + } + + return c, nil +} + +func serializeHTLCAttemptInfo(w io.Writer, a *HTLCAttemptInfo) error { + if err := WriteElements(w, a.sessionKey); err != nil { + return err + } + + if err := SerializeRoute(w, a.Route); err != nil { + return err + } + + if err := serializeTime(w, a.AttemptTime); err != nil { + return err + } + + // If the hash is nil we can just return. + if a.Hash == nil { + return nil + } + + if _, err := w.Write(a.Hash[:]); err != nil { + return err + } + + // Merge the fixed/known records together with the custom records to + // serialize them as a single blob. We can't do this in SerializeRoute + // because we're in the middle of the byte stream there. We can only do + // TLV serialization at the end of the stream, since EOF is allowed for + // a stream if no more data is expected. + producers := []tlv.RecordProducer{ + &a.Route.FirstHopAmount, + } + tlvData, err := lnwire.MergeAndEncode( + producers, nil, a.Route.FirstHopWireCustomRecords, + ) + if err != nil { + return err + } + + if _, err := w.Write(tlvData); err != nil { + return err + } + + return nil +} + +func deserializeHTLCAttemptInfo(r io.Reader) (*HTLCAttemptInfo, error) { + a := &HTLCAttemptInfo{} + err := ReadElements(r, &a.sessionKey) + if err != nil { + return nil, err + } + + a.Route, err = DeserializeRoute(r) + if err != nil { + return nil, err + } + + a.AttemptTime, err = deserializeTime(r) + if err != nil { + return nil, err + } + + hash := lntypes.Hash{} + _, err = io.ReadFull(r, hash[:]) + + switch { + // Older payment attempts wouldn't have the hash set, in which case we + // can just return. + case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): + return a, nil + + case err != nil: + return nil, err + + default: + } + + a.Hash = &hash + + // Read any remaining data (if any) and parse it into the known records + // and custom records. + extraData, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + customRecords, _, _, err := lnwire.ParseAndExtractCustomRecords( + extraData, &a.Route.FirstHopAmount, + ) + if err != nil { + return nil, err + } + + a.Route.FirstHopWireCustomRecords = customRecords + + return a, nil +} + +func serializeHop(w io.Writer, h *Hop) error { + if err := WriteElements(w, + h.PubKeyBytes[:], + h.ChannelID, + h.OutgoingTimeLock, + h.AmtToForward, + ); err != nil { + return err + } + + if err := binary.Write(w, byteOrder, h.LegacyPayload); err != nil { + return err + } + + // For legacy payloads, we don't need to write any TLV records, so + // we'll write a zero indicating the our serialized TLV map has no + // records. + if h.LegacyPayload { + return WriteElements(w, uint32(0)) + } + + // Gather all non-primitive TLV records so that they can be serialized + // as a single blob. + // + // TODO(conner): add migration to unify all fields in a single TLV + // blobs. The split approach will cause headaches down the road as more + // fields are added, which we can avoid by having a single TLV stream + // for all payload fields. + var records []tlv.Record + if h.MPP != nil { + records = append(records, h.MPP.Record()) + } + + // Add blinding point and encrypted data if present. + if h.EncryptedData != nil { + records = append(records, record.NewEncryptedDataRecord( + &h.EncryptedData, + )) + } + + if h.BlindingPoint != nil { + records = append(records, record.NewBlindingPointRecord( + &h.BlindingPoint, + )) + } + + if h.AMP != nil { + records = append(records, h.AMP.Record()) + } + + if h.Metadata != nil { + records = append(records, record.NewMetadataRecord(&h.Metadata)) + } + + if h.TotalAmtMsat != 0 { + totalMsatInt := uint64(h.TotalAmtMsat) + records = append( + records, record.NewTotalAmtMsatBlinded(&totalMsatInt), + ) + } + + // Final sanity check to absolutely rule out custom records that are not + // custom and write into the standard range. + if err := h.CustomRecords.Validate(); err != nil { + return err + } + + // Convert custom records to tlv and add to the record list. + // MapToRecords sorts the list, so adding it here will keep the list + // canonical. + tlvRecords := tlv.MapToRecords(h.CustomRecords) + records = append(records, tlvRecords...) + + // Otherwise, we'll transform our slice of records into a map of the + // raw bytes, then serialize them in-line with a length (number of + // elements) prefix. + mapRecords, err := tlv.RecordsToMap(records) + if err != nil { + return err + } + + numRecords := uint32(len(mapRecords)) + if err := WriteElements(w, numRecords); err != nil { + return err + } + + for recordType, rawBytes := range mapRecords { + if err := WriteElements(w, recordType); err != nil { + return err + } + + if err := wire.WriteVarBytes(w, 0, rawBytes); err != nil { + return err + } + } + + return nil +} + +// maxOnionPayloadSize is the largest Sphinx payload possible, so we don't need +// to read/write a TLV stream larger than this. +const maxOnionPayloadSize = 1300 + +func deserializeHop(r io.Reader) (*Hop, error) { + h := &Hop{} + + var pub []byte + if err := ReadElements(r, &pub); err != nil { + return nil, err + } + copy(h.PubKeyBytes[:], pub) + + if err := ReadElements(r, + &h.ChannelID, &h.OutgoingTimeLock, &h.AmtToForward, + ); err != nil { + return nil, err + } + + // TODO(roasbeef): change field to allow LegacyPayload false to be the + // legacy default? + err := binary.Read(r, byteOrder, &h.LegacyPayload) + if err != nil { + return nil, err + } + + var numElements uint32 + if err := ReadElements(r, &numElements); err != nil { + return nil, err + } + + // If there're no elements, then we can return early. + if numElements == 0 { + return h, nil + } + + tlvMap := make(map[uint64][]byte) + for i := uint32(0); i < numElements; i++ { + var tlvType uint64 + if err := ReadElements(r, &tlvType); err != nil { + return nil, err + } + + rawRecordBytes, err := wire.ReadVarBytes( + r, 0, maxOnionPayloadSize, "tlv", + ) + if err != nil { + return nil, err + } + + tlvMap[tlvType] = rawRecordBytes + } + + // If the MPP type is present, remove it from the generic TLV map and + // parse it back into a proper MPP struct. + // + // TODO(conner): add migration to unify all fields in a single TLV + // blobs. The split approach will cause headaches down the road as more + // fields are added, which we can avoid by having a single TLV stream + // for all payload fields. + mppType := uint64(record.MPPOnionType) + if mppBytes, ok := tlvMap[mppType]; ok { + delete(tlvMap, mppType) + + var ( + mpp = &record.MPP{} + mppRec = mpp.Record() + r = bytes.NewReader(mppBytes) + ) + err := mppRec.Decode(r, uint64(len(mppBytes))) + if err != nil { + return nil, err + } + h.MPP = mpp + } + + // If encrypted data or blinding key are present, remove them from + // the TLV map and parse into proper types. + encryptedDataType := uint64(record.EncryptedDataOnionType) + if data, ok := tlvMap[encryptedDataType]; ok { + delete(tlvMap, encryptedDataType) + h.EncryptedData = data + } + + blindingType := uint64(record.BlindingPointOnionType) + if blindingPoint, ok := tlvMap[blindingType]; ok { + delete(tlvMap, blindingType) + + h.BlindingPoint, err = btcec.ParsePubKey(blindingPoint) + if err != nil { + return nil, fmt.Errorf("invalid blinding point: %w", + err) + } + } + + ampType := uint64(record.AMPOnionType) + if ampBytes, ok := tlvMap[ampType]; ok { + delete(tlvMap, ampType) + + var ( + amp = &record.AMP{} + ampRec = amp.Record() + r = bytes.NewReader(ampBytes) + ) + err := ampRec.Decode(r, uint64(len(ampBytes))) + if err != nil { + return nil, err + } + h.AMP = amp + } + + // If the metadata type is present, remove it from the tlv map and + // populate directly on the hop. + metadataType := uint64(record.MetadataOnionType) + if metadata, ok := tlvMap[metadataType]; ok { + delete(tlvMap, metadataType) + + h.Metadata = metadata + } + + totalAmtMsatType := uint64(record.TotalAmtMsatBlindedType) + if totalAmtMsat, ok := tlvMap[totalAmtMsatType]; ok { + delete(tlvMap, totalAmtMsatType) + + var ( + totalAmtMsatInt uint64 + buf [8]byte + ) + if err := tlv.DTUint64( + bytes.NewReader(totalAmtMsat), + &totalAmtMsatInt, + &buf, + uint64(len(totalAmtMsat)), + ); err != nil { + return nil, err + } + + h.TotalAmtMsat = lnwire.MilliSatoshi(totalAmtMsatInt) + } + + h.CustomRecords = tlvMap + + return h, nil +} + +// SerializeRoute serializes a route. +func SerializeRoute(w io.Writer, r Route) error { + if err := WriteElements(w, + r.TotalTimeLock, r.TotalAmount, r.SourcePubKey[:], + ); err != nil { + return err + } + + if err := WriteElements(w, uint32(len(r.Hops))); err != nil { + return err + } + + for _, h := range r.Hops { + if err := serializeHop(w, h); err != nil { + return err + } + } + + // Any new/extra TLV data is encoded in serializeHTLCAttemptInfo! + + return nil +} + +// DeserializeRoute deserializes a route. +func DeserializeRoute(r io.Reader) (Route, error) { + rt := Route{} + if err := ReadElements(r, + &rt.TotalTimeLock, &rt.TotalAmount, + ); err != nil { + return rt, err + } + + var pub []byte + if err := ReadElements(r, &pub); err != nil { + return rt, err + } + copy(rt.SourcePubKey[:], pub) + + var numHops uint32 + if err := ReadElements(r, &numHops); err != nil { + return rt, err + } + + var hops []*Hop + for i := uint32(0); i < numHops; i++ { + hop, err := deserializeHop(r) + if err != nil { + return rt, err + } + hops = append(hops, hop) + } + rt.Hops = hops + + // Any new/extra TLV data is decoded in deserializeHTLCAttemptInfo! + + return rt, nil +} + +// serializeHTLCSettleInfo serializes the details of a settled htlc. +func serializeHTLCSettleInfo(w io.Writer, s *HTLCSettleInfo) error { + if _, err := w.Write(s.Preimage[:]); err != nil { + return err + } + + if err := serializeTime(w, s.SettleTime); err != nil { + return err + } + + return nil +} + +// deserializeHTLCSettleInfo deserializes the details of a settled htlc. +func deserializeHTLCSettleInfo(r io.Reader) (*HTLCSettleInfo, error) { + s := &HTLCSettleInfo{} + if _, err := io.ReadFull(r, s.Preimage[:]); err != nil { + return nil, err + } + + var err error + s.SettleTime, err = deserializeTime(r) + if err != nil { + return nil, err + } + + return s, nil +} + +// serializeHTLCFailInfo serializes the details of a failed htlc including the +// wire failure. +func serializeHTLCFailInfo(w io.Writer, f *HTLCFailInfo) error { + if err := serializeTime(w, f.FailTime); err != nil { + return err + } + + // Write failure. If there is no failure message, write an empty + // byte slice. + var messageBytes bytes.Buffer + if f.Message != nil { + err := lnwire.EncodeFailureMessage(&messageBytes, f.Message, 0) + if err != nil { + return err + } + } + if err := wire.WriteVarBytes(w, 0, messageBytes.Bytes()); err != nil { + return err + } + + return WriteElements(w, byte(f.Reason), f.FailureSourceIndex) +} + +// deserializeHTLCFailInfo deserializes the details of a failed htlc including +// the wire failure. +func deserializeHTLCFailInfo(r io.Reader) (*HTLCFailInfo, error) { + f := &HTLCFailInfo{} + var err error + f.FailTime, err = deserializeTime(r) + if err != nil { + return nil, err + } + + // Read failure. + failureBytes, err := wire.ReadVarBytes( + r, 0, math.MaxUint16, "failure", + ) + if err != nil { + return nil, err + } + if len(failureBytes) > 0 { + f.Message, err = lnwire.DecodeFailureMessage( + bytes.NewReader(failureBytes), 0, + ) + if err != nil && + !errors.Is(err, lnwire.ErrParsingExtraTLVBytes) { + + return nil, err + } + + // In case we have an invalid TLV stream regarding the extra + // tlv data we still continue with the decoding of the + // HTLCFailInfo. + if errors.Is(err, lnwire.ErrParsingExtraTLVBytes) { + log.Warnf("Failed to decode extra TLV bytes for "+ + "failure message: %v", err) + } + } + + var reason byte + err = ReadElements(r, &reason, &f.FailureSourceIndex) + if err != nil { + return nil, err + } + f.Reason = HTLCFailReason(reason) + + return f, nil +} diff --git a/payments/db/migration1/lnwire/channel_id.go b/payments/db/migration1/lnwire/channel_id.go new file mode 100644 index 000000000..5c9eca34f --- /dev/null +++ b/payments/db/migration1/lnwire/channel_id.go @@ -0,0 +1,127 @@ +package lnwire + +import ( + "encoding/binary" + "encoding/hex" + "io" + "math" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // MaxFundingTxOutputs is the maximum number of allowed outputs on a + // funding transaction within the protocol. This is due to the fact + // that we use 2-bytes to encode the index within the funding output + // during the funding workflow. Funding transaction with more outputs + // than this are considered invalid within the protocol. + MaxFundingTxOutputs = math.MaxUint16 +) + +// ChannelID is a series of 32-bytes that uniquely identifies all channels +// within the network. The ChannelID is computed using the outpoint of the +// funding transaction (the txid, and output index). Given a funding output the +// ChannelID can be calculated by XOR'ing the big-endian serialization of the +// txid and the big-endian serialization of the output index, truncated to +// 2 bytes. +type ChannelID [32]byte + +// ConnectionWideID is an all-zero ChannelID, which is used to represent a +// message intended for all channels to specific peer. +var ConnectionWideID = ChannelID{} + +// String returns the string representation of the ChannelID. This is just the +// hex string encoding of the ChannelID itself. +func (c ChannelID) String() string { + return hex.EncodeToString(c[:]) +} + +// Record returns a TLV record that can be used to encode/decode a ChannelID +// to/from a TLV stream. +func (c *ChannelID) Record() tlv.Record { + return tlv.MakeStaticRecord(0, c, 32, encodeChannelID, decodeChannelID) +} + +func encodeChannelID(w io.Writer, val interface{}, buf *[8]byte) error { + if v, ok := val.(*ChannelID); ok { + bigSize := [32]byte(*v) + + return tlv.EBytes32(w, &bigSize, buf) + } + + return tlv.NewTypeForEncodingErr(val, "lnwire.ChannelID") +} + +func decodeChannelID(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if v, ok := val.(*ChannelID); ok { + var id [32]byte + err := tlv.DBytes32(r, &id, buf, l) + if err != nil { + return err + } + + *v = id + + return nil + } + + return tlv.NewTypeForDecodingErr(val, "lnwire.ChannelID", l, l) +} + +// NewChanIDFromOutPoint converts a target OutPoint into a ChannelID that is +// usable within the network. In order to convert the OutPoint into a ChannelID, +// we XOR the lower 2-bytes of the txid within the OutPoint with the big-endian +// serialization of the Index of the OutPoint, truncated to 2-bytes. +func NewChanIDFromOutPoint(op wire.OutPoint) ChannelID { + // First we'll copy the txid of the outpoint into our channel ID slice. + var cid ChannelID + copy(cid[:], op.Hash[:]) + + // With the txid copied over, we'll now XOR the lower 2-bytes of the + // partial channelID with big-endian serialization of output index. + xorTxid(&cid, uint16(op.Index)) + + return cid +} + +// xorTxid performs the transformation needed to transform an OutPoint into a +// ChannelID. To do this, we expect the cid parameter to contain the txid +// unaltered and the outputIndex to be the output index +func xorTxid(cid *ChannelID, outputIndex uint16) { + var buf [2]byte + binary.BigEndian.PutUint16(buf[:], outputIndex) + + cid[30] ^= buf[0] + cid[31] ^= buf[1] +} + +// GenPossibleOutPoints generates all the possible outputs given a channel ID. +// In order to generate these possible outpoints, we perform a brute-force +// search through the candidate output index space, performing a reverse +// mapping from channelID back to OutPoint. +func (c *ChannelID) GenPossibleOutPoints() [MaxFundingTxOutputs]wire.OutPoint { + var possiblePoints [MaxFundingTxOutputs]wire.OutPoint + for i := uint16(0); i < MaxFundingTxOutputs; i++ { + cidCopy := *c + xorTxid(&cidCopy, i) + + possiblePoints[i] = wire.OutPoint{ + Hash: chainhash.Hash(cidCopy), + Index: uint32(i), + } + } + + return possiblePoints +} + +// IsChanPoint returns true if the OutPoint passed corresponds to the target +// ChannelID. +func (c ChannelID) IsChanPoint(op *wire.OutPoint) bool { + candidateCid := NewChanIDFromOutPoint(*op) + + return candidateCid == c +} diff --git a/payments/db/migration1/lnwire/channel_update.go b/payments/db/migration1/lnwire/channel_update.go new file mode 100644 index 000000000..09b8044bd --- /dev/null +++ b/payments/db/migration1/lnwire/channel_update.go @@ -0,0 +1,422 @@ +package lnwire + +import ( + "bytes" + "fmt" + "io" + + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/lightningnetwork/lnd/tlv" +) + +// ChanUpdateMsgFlags is a bitfield that signals whether optional fields are +// present in the ChannelUpdate. +type ChanUpdateMsgFlags uint8 + +const ( + // ChanUpdateRequiredMaxHtlc is a bit that indicates whether the + // required htlc_maximum_msat field is present in this ChannelUpdate. + ChanUpdateRequiredMaxHtlc ChanUpdateMsgFlags = 1 << iota +) + +// String returns the bitfield flags as a string. +func (c ChanUpdateMsgFlags) String() string { + return fmt.Sprintf("%08b", c) +} + +// HasMaxHtlc returns true if the htlc_maximum_msat option bit is set in the +// message flags. +func (c ChanUpdateMsgFlags) HasMaxHtlc() bool { + return c&ChanUpdateRequiredMaxHtlc != 0 +} + +// ChanUpdateChanFlags is a bitfield that signals various options concerning a +// particular channel edge. Each bit is to be examined in order to determine +// how the ChannelUpdate message is to be interpreted. +type ChanUpdateChanFlags uint8 + +const ( + // ChanUpdateDirection indicates the direction of a channel update. If + // this bit is set to 0 if Node1 (the node with the "smaller" Node ID) + // is updating the channel, and to 1 otherwise. + ChanUpdateDirection ChanUpdateChanFlags = 1 << iota + + // ChanUpdateDisabled is a bit that indicates if the channel edge + // selected by the ChanUpdateDirection bit is to be treated as being + // disabled. + ChanUpdateDisabled +) + +// IsDisabled determines whether the channel flags has the disabled bit set. +func (c ChanUpdateChanFlags) IsDisabled() bool { + return c&ChanUpdateDisabled == ChanUpdateDisabled +} + +// String returns the bitfield flags as a string. +func (c ChanUpdateChanFlags) String() string { + return fmt.Sprintf("%08b", c) +} + +// ChannelUpdate1 message is used after channel has been initially announced. +// Each side independently announces its fees and minimum expiry for HTLCs and +// other parameters. Also this message is used to redeclare initially set +// channel parameters. +type ChannelUpdate1 struct { + // Signature is used to validate the announced data and prove the + // ownership of node id. + Signature Sig + + // ChainHash denotes the target chain that this channel was opened + // within. This value should be the genesis hash of the target chain. + // Along with the short channel ID, this uniquely identifies the + // channel globally in a blockchain. + ChainHash chainhash.Hash + + // ShortChannelID is the unique description of the funding transaction. + ShortChannelID ShortChannelID + + // Timestamp allows ordering in the case of multiple announcements. We + // should ignore the message if timestamp is not greater than + // the last-received. + Timestamp uint32 + + // MessageFlags is a bitfield that describes whether optional fields + // are present in this update. Currently, the least-significant bit + // must be set to 1 if the optional field MaxHtlc is present. + MessageFlags ChanUpdateMsgFlags + + // ChannelFlags is a bitfield that describes additional meta-data + // concerning how the update is to be interpreted. Currently, the + // least-significant bit must be set to 0 if the creating node + // corresponds to the first node in the previously sent channel + // announcement and 1 otherwise. If the second bit is set, then the + // channel is set to be disabled. + ChannelFlags ChanUpdateChanFlags + + // TimeLockDelta is the minimum number of blocks this node requires to + // be added to the expiry of HTLCs. This is a security parameter + // determined by the node operator. This value represents the required + // gap between the time locks of the incoming and outgoing HTLC's set + // to this node. + TimeLockDelta uint16 + + // HtlcMinimumMsat is the minimum HTLC value which will be accepted. + HtlcMinimumMsat MilliSatoshi + + // BaseFee is the base fee that must be used for incoming HTLC's to + // this particular channel. This value will be tacked onto the required + // for a payment independent of the size of the payment. + BaseFee uint32 + + // FeeRate is the fee rate that will be charged per millionth of a + // satoshi. + FeeRate uint32 + + // HtlcMaximumMsat is the maximum HTLC value which will be accepted. + HtlcMaximumMsat MilliSatoshi + + // InboundFee is an optional TLV record that contains the fee + // information for incoming HTLCs. + InboundFee tlv.OptionalRecordT[tlv.TlvType55555, Fee] + + // ExtraData is the set of data that was appended to this message to + // fill out the full maximum transport message size. These fields can + // be used to specify optional data such as custom TLV fields. + ExtraOpaqueData ExtraOpaqueData +} + +// A compile time check to ensure ChannelUpdate implements the lnwire.Message +// interface. +var _ Message = (*ChannelUpdate1)(nil) + +// A compile time check to ensure ChannelUpdate1 implements the +// lnwire.SizeableMessage interface. +var _ SizeableMessage = (*ChannelUpdate1)(nil) + +// Decode deserializes a serialized ChannelUpdate stored in the passed +// io.Reader observing the specified protocol version. +// +// This is part of the lnwire.Message interface. +func (a *ChannelUpdate1) Decode(r io.Reader, _ uint32) error { + err := ReadElements(r, + &a.Signature, + a.ChainHash[:], + &a.ShortChannelID, + &a.Timestamp, + &a.MessageFlags, + &a.ChannelFlags, + &a.TimeLockDelta, + &a.HtlcMinimumMsat, + &a.BaseFee, + &a.FeeRate, + ) + if err != nil { + return err + } + + // Now check whether the max HTLC field is present and read it if so. + if a.MessageFlags.HasMaxHtlc() { + if err := ReadElements(r, &a.HtlcMaximumMsat); err != nil { + return err + } + } + + var tlvRecords ExtraOpaqueData + if err := ReadElements(r, &tlvRecords); err != nil { + return err + } + + var inboundFee = a.InboundFee.Zero() + typeMap, err := tlvRecords.ExtractRecords(&inboundFee) + if err != nil { + return fmt.Errorf("%w: %w", ErrParsingExtraTLVBytes, err) + } + + val, ok := typeMap[a.InboundFee.TlvType()] + if ok && val == nil { + a.InboundFee = tlv.SomeRecordT(inboundFee) + } + + if len(tlvRecords) != 0 { + a.ExtraOpaqueData = tlvRecords + } + + return nil +} + +// Encode serializes the target ChannelUpdate into the passed io.Writer +// observing the protocol version specified. +// +// This is part of the lnwire.Message interface. +func (a *ChannelUpdate1) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteSig(w, a.Signature); err != nil { + return err + } + + if err := WriteBytes(w, a.ChainHash[:]); err != nil { + return err + } + + if err := WriteShortChannelID(w, a.ShortChannelID); err != nil { + return err + } + + if err := WriteUint32(w, a.Timestamp); err != nil { + return err + } + + if err := WriteChanUpdateMsgFlags(w, a.MessageFlags); err != nil { + return err + } + + if err := WriteChanUpdateChanFlags(w, a.ChannelFlags); err != nil { + return err + } + + if err := WriteUint16(w, a.TimeLockDelta); err != nil { + return err + } + + if err := WriteMilliSatoshi(w, a.HtlcMinimumMsat); err != nil { + return err + } + + if err := WriteUint32(w, a.BaseFee); err != nil { + return err + } + + if err := WriteUint32(w, a.FeeRate); err != nil { + return err + } + + // Now append optional fields if they are set. Currently, the only + // optional field is max HTLC. + if a.MessageFlags.HasMaxHtlc() { + err := WriteMilliSatoshi(w, a.HtlcMaximumMsat) + if err != nil { + return err + } + } + + recordProducers := make([]tlv.RecordProducer, 0, 1) + a.InboundFee.WhenSome(func(fee tlv.RecordT[tlv.TlvType55555, Fee]) { + recordProducers = append(recordProducers, &fee) + }) + + err := EncodeMessageExtraData(&a.ExtraOpaqueData, recordProducers...) + if err != nil { + return err + } + + // Finally, append any extra opaque data. + return WriteBytes(w, a.ExtraOpaqueData) +} + +// MsgType returns the integer uniquely identifying this message type on the +// wire. +// +// This is part of the lnwire.Message interface. +func (a *ChannelUpdate1) MsgType() MessageType { + return MsgChannelUpdate +} + +// DataToSign is used to retrieve part of the announcement message which should +// be signed. +func (a *ChannelUpdate1) DataToSign() ([]byte, error) { + // We should not include the signatures itself. + b := make([]byte, 0, MaxMsgBody) + buf := bytes.NewBuffer(b) + if err := WriteBytes(buf, a.ChainHash[:]); err != nil { + return nil, err + } + + if err := WriteShortChannelID(buf, a.ShortChannelID); err != nil { + return nil, err + } + + if err := WriteUint32(buf, a.Timestamp); err != nil { + return nil, err + } + + if err := WriteChanUpdateMsgFlags(buf, a.MessageFlags); err != nil { + return nil, err + } + + if err := WriteChanUpdateChanFlags(buf, a.ChannelFlags); err != nil { + return nil, err + } + + if err := WriteUint16(buf, a.TimeLockDelta); err != nil { + return nil, err + } + + if err := WriteMilliSatoshi(buf, a.HtlcMinimumMsat); err != nil { + return nil, err + } + + if err := WriteUint32(buf, a.BaseFee); err != nil { + return nil, err + } + + if err := WriteUint32(buf, a.FeeRate); err != nil { + return nil, err + } + + // Now append optional fields if they are set. Currently, the only + // optional field is max HTLC. + if a.MessageFlags.HasMaxHtlc() { + err := WriteMilliSatoshi(buf, a.HtlcMaximumMsat) + if err != nil { + return nil, err + } + } + + // Finally, append any extra opaque data. + if err := WriteBytes(buf, a.ExtraOpaqueData); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// SCID returns the ShortChannelID of the channel that the update applies to. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) SCID() ShortChannelID { + return a.ShortChannelID +} + +// IsNode1 is true if the update was produced by node 1 of the channel peers. +// Node 1 is the node with the lexicographically smaller public key. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) IsNode1() bool { + return a.ChannelFlags&ChanUpdateDirection == 0 +} + +// IsDisabled is true if the update is announcing that the channel should be +// considered disabled. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) IsDisabled() bool { + return a.ChannelFlags&ChanUpdateDisabled == ChanUpdateDisabled +} + +// GetChainHash returns the hash of the chain that the message is referring to. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) GetChainHash() chainhash.Hash { + return a.ChainHash +} + +// ForwardingPolicy returns the set of forwarding constraints of the update. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) ForwardingPolicy() *ForwardingPolicy { + return &ForwardingPolicy{ + TimeLockDelta: a.TimeLockDelta, + BaseFee: MilliSatoshi(a.BaseFee), + FeeRate: MilliSatoshi(a.FeeRate), + MinHTLC: a.HtlcMinimumMsat, + HasMaxHTLC: a.MessageFlags.HasMaxHtlc(), + MaxHTLC: a.HtlcMaximumMsat, + } +} + +// GossipVersion returns the gossip version that this message is part of. +// +// NOTE: this is part of the GossipMessage interface. +func (a *ChannelUpdate1) GossipVersion() GossipVersion { + return GossipVersion1 +} + +// CmpAge can be used to determine if the update is older or newer than the +// passed update. It returns 1 if this update is newer, -1 if it is older, and +// 0 if they are the same age. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) CmpAge(update ChannelUpdate) (CompareResult, error) { + other, ok := update.(*ChannelUpdate1) + if !ok { + return 0, fmt.Errorf("expected *ChannelUpdate1, got: %T", + update) + } + + switch { + case a.Timestamp > other.Timestamp: + return GreaterThan, nil + case a.Timestamp < other.Timestamp: + return LessThan, nil + default: + return EqualTo, nil + } +} + +// SetDisabledFlag can be used to adjust the disabled flag of an update. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) SetDisabledFlag(disabled bool) { + if disabled { + a.ChannelFlags |= ChanUpdateDisabled + } else { + a.ChannelFlags &= ^ChanUpdateDisabled + } +} + +// SetSCID can be used to overwrite the SCID of the update. +// +// NOTE: this is part of the ChannelUpdate interface. +func (a *ChannelUpdate1) SetSCID(scid ShortChannelID) { + a.ShortChannelID = scid +} + +// A compile time assertion to ensure ChannelUpdate1 implements the +// ChannelUpdate interface. +var _ ChannelUpdate = (*ChannelUpdate1)(nil) + +// SerializedSize returns the serialized size of the message in bytes. +// +// This is part of the lnwire.SizeableMessage interface. +func (a *ChannelUpdate1) SerializedSize() (uint32, error) { + return MessageSerializedSize(a) +} diff --git a/payments/db/migration1/lnwire/custom_records.go b/payments/db/migration1/lnwire/custom_records.go new file mode 100644 index 000000000..de5ff4a23 --- /dev/null +++ b/payments/db/migration1/lnwire/custom_records.go @@ -0,0 +1,278 @@ +package lnwire + +import ( + "bytes" + "fmt" + "io" + "maps" + "sort" + + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // MinCustomRecordsTlvType is the minimum custom records TLV type as + // defined in BOLT 01. + MinCustomRecordsTlvType = 65536 +) + +// CustomRecords stores a set of custom key/value pairs. Map keys are TLV types +// which must be greater than or equal to MinCustomRecordsTlvType. +type CustomRecords map[uint64][]byte + +// NewCustomRecords creates a new CustomRecords instance from a +// tlv.TypeMap. +func NewCustomRecords(tlvMap tlv.TypeMap) (CustomRecords, error) { + // Make comparisons in unit tests easy by returning nil if the map is + // empty. + if len(tlvMap) == 0 { + return nil, nil + } + + customRecords := make(CustomRecords, len(tlvMap)) + for k, v := range tlvMap { + customRecords[uint64(k)] = v + } + + // Validate the custom records. + err := customRecords.Validate() + if err != nil { + return nil, fmt.Errorf("custom records from tlv map "+ + "validation error: %w", err) + } + + return customRecords, nil +} + +// ParseCustomRecords creates a new CustomRecords instance from a tlv.Blob. +func ParseCustomRecords(b tlv.Blob) (CustomRecords, error) { + return ParseCustomRecordsFrom(bytes.NewReader(b)) +} + +// ParseCustomRecordsFrom creates a new CustomRecords instance from a reader. +func ParseCustomRecordsFrom(r io.Reader) (CustomRecords, error) { + typeMap, err := DecodeRecords(r) + if err != nil { + return nil, fmt.Errorf("error decoding HTLC record: %w", err) + } + + return NewCustomRecords(typeMap) +} + +// Validate checks that all custom records are in the custom type range. +func (c CustomRecords) Validate() error { + if c == nil { + return nil + } + + for key := range c { + if key < MinCustomRecordsTlvType { + return fmt.Errorf("custom records entry with TLV "+ + "type below min: %d", MinCustomRecordsTlvType) + } + } + + return nil +} + +// Copy returns a copy of the custom records. +func (c CustomRecords) Copy() CustomRecords { + if c == nil { + return nil + } + + customRecords := make(CustomRecords, len(c)) + for k, v := range c { + customRecords[k] = v + } + + return customRecords +} + +// MergedCopy creates a copy of the records and merges them with the given +// records. If the same key is present in both sets, the value from the other +// records will be used. +func (c CustomRecords) MergedCopy(other CustomRecords) CustomRecords { + copiedRecords := make(CustomRecords, len(c)) + maps.Copy(copiedRecords, c) + maps.Copy(copiedRecords, other) + + return copiedRecords +} + +// ExtendRecordProducers extends the given records slice with the custom +// records. The resultant records slice will be sorted if the given records +// slice contains TLV types greater than or equal to MinCustomRecordsTlvType. +func (c CustomRecords) ExtendRecordProducers( + producers []tlv.RecordProducer) ([]tlv.RecordProducer, error) { + + // If the custom records are nil or empty, there is nothing to do. + if len(c) == 0 { + return producers, nil + } + + // Validate the custom records. + err := c.Validate() + if err != nil { + return nil, err + } + + // Ensure that the existing records slice TLV types are not also present + // in the custom records. If they are, the resultant extended records + // slice would erroneously contain duplicate TLV types. + for _, rp := range producers { + record := rp.Record() + recordTlvType := uint64(record.Type()) + + _, foundDuplicateTlvType := c[recordTlvType] + if foundDuplicateTlvType { + return nil, fmt.Errorf("custom records contains a TLV "+ + "type that is already present in the "+ + "existing records: %d", recordTlvType) + } + } + + // Convert the custom records map to a TLV record producer slice and + // append them to the exiting records slice. + customRecordProducers := RecordsAsProducers(tlv.MapToRecords(c)) + producers = append(producers, customRecordProducers...) + + // If the records slice which was given as an argument included TLV + // values greater than or equal to the minimum custom records TLV type + // we will sort the extended records slice to ensure that it is ordered + // correctly. + SortProducers(producers) + + return producers, nil +} + +// RecordProducers returns a slice of record producers for the custom records. +func (c CustomRecords) RecordProducers() []tlv.RecordProducer { + // If the custom records are nil or empty, return an empty slice. + if len(c) == 0 { + return nil + } + + // Convert the custom records map to a TLV record producer slice. + records := tlv.MapToRecords(c) + + return RecordsAsProducers(records) +} + +// Serialize serializes the custom records into a byte slice. +func (c CustomRecords) Serialize() ([]byte, error) { + records := tlv.MapToRecords(c) + return EncodeRecords(records) +} + +// SerializeTo serializes the custom records into the given writer. +func (c CustomRecords) SerializeTo(w io.Writer) error { + records := tlv.MapToRecords(c) + return EncodeRecordsTo(w, records) +} + +// ProduceRecordsSorted converts a slice of record producers into a slice of +// records and then sorts it by type. +func ProduceRecordsSorted(recordProducers ...tlv.RecordProducer) []tlv.Record { + records := fn.Map( + recordProducers, + func(producer tlv.RecordProducer) tlv.Record { + return producer.Record() + }, + ) + + // Ensure that the set of records are sorted before we attempt to + // decode from the stream, to ensure they're canonical. + tlv.SortRecords(records) + + return records +} + +// SortProducers sorts the given record producers by their type. +func SortProducers(producers []tlv.RecordProducer) { + sort.Slice(producers, func(i, j int) bool { + recordI := producers[i].Record() + recordJ := producers[j].Record() + return recordI.Type() < recordJ.Type() + }) +} + +// TlvMapToRecords converts a TLV map into a slice of records. +func TlvMapToRecords(tlvMap tlv.TypeMap) []tlv.Record { + tlvMapGeneric := make(map[uint64][]byte) + for k, v := range tlvMap { + tlvMapGeneric[uint64(k)] = v + } + + return tlv.MapToRecords(tlvMapGeneric) +} + +// RecordsAsProducers converts a slice of records into a slice of record +// producers. +func RecordsAsProducers(records []tlv.Record) []tlv.RecordProducer { + return fn.Map(records, func(record tlv.Record) tlv.RecordProducer { + return &record + }) +} + +// EncodeRecords encodes the given records into a byte slice. +func EncodeRecords(records []tlv.Record) ([]byte, error) { + var buf bytes.Buffer + if err := EncodeRecordsTo(&buf, records); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// EncodeRecordsTo encodes the given records into the given writer. +func EncodeRecordsTo(w io.Writer, records []tlv.Record) error { + tlvStream, err := tlv.NewStream(records...) + if err != nil { + return err + } + + return tlvStream.Encode(w) +} + +// DecodeRecords decodes the given byte slice into the given records and returns +// the rest as a TLV type map. +func DecodeRecords(r io.Reader, + records ...tlv.Record) (tlv.TypeMap, error) { + + tlvStream, err := tlv.NewStream(records...) + if err != nil { + return nil, err + } + + return tlvStream.DecodeWithParsedTypes(r) +} + +// DecodeRecordsP2P decodes the given byte slice into the given records and +// returns the rest as a TLV type map. This function is identical to +// DecodeRecords except that the record size is capped at 65535. +func DecodeRecordsP2P(r *bytes.Reader, + records ...tlv.Record) (tlv.TypeMap, error) { + + tlvStream, err := tlv.NewStream(records...) + if err != nil { + return nil, err + } + + return tlvStream.DecodeWithParsedTypesP2P(r) +} + +// AssertUniqueTypes asserts that the given records have unique types. +func AssertUniqueTypes(r []tlv.Record) error { + seen := make(fn.Set[tlv.Type], len(r)) + for _, record := range r { + t := record.Type() + if seen.Contains(t) { + return fmt.Errorf("duplicate record type: %d", t) + } + seen.Add(t) + } + + return nil +} diff --git a/payments/db/migration1/lnwire/error.go b/payments/db/migration1/lnwire/error.go new file mode 100644 index 000000000..382411999 --- /dev/null +++ b/payments/db/migration1/lnwire/error.go @@ -0,0 +1,143 @@ +package lnwire + +import ( + "bytes" + "fmt" + "io" +) + +var ( + // ErrParsingExtraTLVBytes is returned when we attempt to parse + // extra opaque bytes as a TLV stream, but the parsing fails due to + // and invalid TLV stream. + ErrParsingExtraTLVBytes = fmt.Errorf("error parsing extra TLV bytes") +) + +// FundingError represents a set of errors that can be encountered and sent +// during the funding workflow. +type FundingError uint8 + +const ( + // ErrMaxPendingChannels is returned by remote peer when the number of + // active pending channels exceeds their maximum policy limit. + ErrMaxPendingChannels FundingError = 1 + + // ErrChanTooLarge is returned by a remote peer that receives a + // FundingOpen request for a channel that is above their current + // soft-limit. + ErrChanTooLarge FundingError = 2 +) + +// String returns a human readable version of the target FundingError. +func (e FundingError) String() string { + switch e { + case ErrMaxPendingChannels: + return "Number of pending channels exceed maximum" + case ErrChanTooLarge: + return "channel too large" + default: + return "unknown error" + } +} + +// Error returns the human readable version of the target FundingError. +// +// NOTE: Satisfies the Error interface. +func (e FundingError) Error() string { + return e.String() +} + +// ErrorData is a set of bytes associated with a particular sent error. A +// receiving node SHOULD only print out data verbatim if the string is composed +// solely of printable ASCII characters. For reference, the printable character +// set includes byte values 32 through 127 inclusive. +type ErrorData []byte + +// Error represents a generic error bound to an exact channel. The message +// format is purposefully general in order to allow expression of a wide array +// of possible errors. Each Error message is directed at a particular open +// channel referenced by ChannelPoint. +type Error struct { + // ChanID references the active channel in which the error occurred + // within. If the ChanID is all zeros, then this error applies to the + // entire established connection. + ChanID ChannelID + + // Data is the attached error data that describes the exact failure + // which caused the error message to be sent. + Data ErrorData +} + +// NewError creates a new Error message. +func NewError() *Error { + return &Error{} +} + +// A compile time check to ensure Error implements the lnwire.Message +// interface. +var _ Message = (*Error)(nil) + +// A compile time check to ensure Error implements the lnwire.SizeableMessage +// interface. +var _ SizeableMessage = (*Error)(nil) + +// Error returns the string representation to Error. +// +// NOTE: Satisfies the error interface. +func (c *Error) Error() string { + errMsg := "non-ascii data" + if isASCII(c.Data) { + errMsg = string(c.Data) + } + + return fmt.Sprintf("chan_id=%v, err=%v", c.ChanID, errMsg) +} + +// Decode deserializes a serialized Error message stored in the passed +// io.Reader observing the specified protocol version. +// +// This is part of the lnwire.Message interface. +func (c *Error) Decode(r io.Reader, pver uint32) error { + return ReadElements(r, + &c.ChanID, + &c.Data, + ) +} + +// Encode serializes the target Error into the passed io.Writer observing the +// protocol version specified. +// +// This is part of the lnwire.Message interface. +func (c *Error) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteBytes(w, c.ChanID[:]); err != nil { + return err + } + + return WriteErrorData(w, c.Data) +} + +// MsgType returns the integer uniquely identifying an Error message on the +// wire. +// +// This is part of the lnwire.Message interface. +func (c *Error) MsgType() MessageType { + return MsgError +} + +// SerializedSize returns the serialized size of the message in bytes. +// +// This is part of the lnwire.SizeableMessage interface. +func (c *Error) SerializedSize() (uint32, error) { + return MessageSerializedSize(c) +} + +// isASCII is a helper method that checks whether all bytes in `data` would be +// printable ASCII characters if interpreted as a string. +func isASCII(data []byte) bool { + for _, c := range data { + if c < 32 || c > 126 { + return false + } + } + return true +} diff --git a/payments/db/migration1/lnwire/extra_bytes.go b/payments/db/migration1/lnwire/extra_bytes.go new file mode 100644 index 000000000..39228c101 --- /dev/null +++ b/payments/db/migration1/lnwire/extra_bytes.go @@ -0,0 +1,309 @@ +package lnwire + +import ( + "bytes" + "fmt" + "io" + + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/tlv" +) + +// ExtraOpaqueData is the set of data that was appended to this message, some +// of which we may not actually know how to iterate or parse. By holding onto +// this data, we ensure that we're able to properly validate the set of +// signatures that cover these new fields, and ensure we're able to make +// upgrades to the network in a forwards compatible manner. +type ExtraOpaqueData []byte + +// NewExtraOpaqueData creates a new ExtraOpaqueData instance from a tlv.TypeMap. +func NewExtraOpaqueData(tlvMap tlv.TypeMap) (ExtraOpaqueData, error) { + // If the tlv map is empty, we'll want to mirror the behavior of + // decoding an empty extra opaque data field (see Decode method). + if len(tlvMap) == 0 { + return make([]byte, 0), nil + } + + // Convert the TLV map into a slice of records. + records := TlvMapToRecords(tlvMap) + + // Encode the records into the extra data byte slice. + return EncodeRecords(records) +} + +// Encode attempts to encode the raw extra bytes into the passed io.Writer. +func (e *ExtraOpaqueData) Encode(w *bytes.Buffer) error { + eBytes := []byte((*e)[:]) + if err := WriteBytes(w, eBytes); err != nil { + return err + } + + return nil +} + +// Decode attempts to unpack the raw bytes encoded in the passed-in io.Reader as +// a set of extra opaque data. +func (e *ExtraOpaqueData) Decode(r io.Reader) error { + // First, we'll attempt to read a set of bytes contained within the + // passed io.Reader (if any exist). + rawBytes, err := io.ReadAll(r) + if err != nil { + return err + } + + // If we _do_ have some bytes, then we'll swap out our backing pointer. + // This ensures that any struct that embeds this type will properly + // store the bytes once this method exits. + if len(rawBytes) > 0 { + *e = rawBytes + } else { + *e = make([]byte, 0) + } + + return nil +} + +// ValidateTLV checks that the raw bytes that make up the ExtraOpaqueData +// instance are a valid TLV stream. +func (e *ExtraOpaqueData) ValidateTLV() error { + // There is nothing to validate if the ExtraOpaqueData is nil or empty. + if e == nil || len(*e) == 0 { + return nil + } + + tlvStream, err := tlv.NewStream() + if err != nil { + return err + } + + // Ensure that the TLV stream is valid by attempting to decode it. + _, err = tlvStream.DecodeWithParsedTypesP2P(bytes.NewReader(*e)) + if err != nil { + return fmt.Errorf("invalid TLV stream: %w: %v", err, *e) + } + + return nil +} + +// PackRecords attempts to encode the set of tlv records into the target +// ExtraOpaqueData instance. The records will be encoded as a raw TLV stream +// and stored within the backing slice pointer. +func (e *ExtraOpaqueData) PackRecords( + recordProducers ...tlv.RecordProducer) error { + + // Assemble all the records passed in series, then encode them. + records := ProduceRecordsSorted(recordProducers...) + encoded, err := EncodeRecords(records) + if err != nil { + return err + } + + *e = encoded + + return nil +} + +// ExtractRecords attempts to decode any types in the internal raw bytes as if +// it were a tlv stream. The set of raw parsed types is returned, and any +// passed records (if found in the stream) will be parsed into the proper +// tlv.Record. +func (e *ExtraOpaqueData) ExtractRecords( + recordProducers ...tlv.RecordProducer) (tlv.TypeMap, error) { + + // First, assemble all the records passed in series. + records := ProduceRecordsSorted(recordProducers...) + extraBytesReader := bytes.NewReader(*e) + + // Since ExtraOpaqueData is provided by a potentially malicious peer, + // pass it into the P2P decoding variant. + return DecodeRecordsP2P(extraBytesReader, records...) +} + +// RecordProducers parses ExtraOpaqueData into a slice of TLV record producers +// by interpreting it as a TLV map. +func (e *ExtraOpaqueData) RecordProducers() ([]tlv.RecordProducer, error) { + var recordProducers []tlv.RecordProducer + + // If the instance is nil or empty, return an empty slice. + if e == nil || len(*e) == 0 { + return recordProducers, nil + } + + // Parse the extra opaque data as a TLV map. + tlvMap, err := e.ExtractRecords() + if err != nil { + return nil, err + } + + // Convert the TLV map into a slice of record producers. + records := TlvMapToRecords(tlvMap) + + return RecordsAsProducers(records), nil +} + +// EncodeMessageExtraData encodes the given recordProducers into the given +// extraData. +func EncodeMessageExtraData(extraData *ExtraOpaqueData, + recordProducers ...tlv.RecordProducer) error { + + // Treat extraData as a mutable reference. + if extraData == nil { + return fmt.Errorf("extra data cannot be nil") + } + + // Pack in the series of TLV records into this message. The order we + // pass them in doesn't matter, as the method will ensure that things + // are all properly sorted. + return extraData.PackRecords(recordProducers...) +} + +// ParseAndExtractCustomRecords parses the given extra data into the passed-in +// records, then returns any remaining records split into custom records and +// extra data. +func ParseAndExtractCustomRecords(allExtraData ExtraOpaqueData, + knownRecords ...tlv.RecordProducer) (CustomRecords, + fn.Set[tlv.Type], ExtraOpaqueData, error) { + + extraDataTlvMap, err := allExtraData.ExtractRecords(knownRecords...) + if err != nil { + return nil, nil, nil, err + } + + // Remove the known and now extracted records from the leftover extra + // data map. + parsedKnownRecords := make(fn.Set[tlv.Type], len(knownRecords)) + for _, producer := range knownRecords { + r := producer.Record() + + // Only remove the records if it was parsed (remainder is nil). + // We'll just store the type so we can tell the caller which + // records were actually parsed fully. + val, ok := extraDataTlvMap[r.Type()] + if ok && val == nil { + parsedKnownRecords.Add(r.Type()) + delete(extraDataTlvMap, r.Type()) + } + } + + // Any records from the extra data TLV map which are in the custom + // records TLV type range will be included in the custom records field + // and removed from the extra data field. + customRecordsTlvMap := make(tlv.TypeMap, len(extraDataTlvMap)) + for k, v := range extraDataTlvMap { + // Skip records that are not in the custom records TLV type + // range. + if k < MinCustomRecordsTlvType { + continue + } + + // Include the record in the custom records map. + customRecordsTlvMap[k] = v + + // Now that the record is included in the custom records map, + // we can remove it from the extra data TLV map. + delete(extraDataTlvMap, k) + } + + // Set the custom records field to the custom records specific TLV + // record map. + customRecords, err := NewCustomRecords(customRecordsTlvMap) + if err != nil { + return nil, nil, nil, err + } + + // Encode the remaining records back into the extra data field. These + // records are not in the custom records TLV type range and do not + // have associated fields in the struct that produced the records. + extraData, err := NewExtraOpaqueData(extraDataTlvMap) + if err != nil { + return nil, nil, nil, err + } + + // Help with unit testing where we might have the empty value (nil) for + // the extra data instead of the default that's returned by the + // constructor (empty slice). + if len(extraData) == 0 { + extraData = nil + } + + return customRecords, parsedKnownRecords, extraData, nil +} + +// MergeAndEncode merges the known records with the extra data and custom +// records, then encodes the merged records into raw bytes. +func MergeAndEncode(knownRecords []tlv.RecordProducer, + extraData ExtraOpaqueData, customRecords CustomRecords) ([]byte, + error) { + + // Construct a slice of all the records that we should include in the + // message extra data field. We will start by including any records from + // the extra data field. + mergedRecords, err := extraData.RecordProducers() + if err != nil { + return nil, err + } + + // Merge the known and extra data records. + mergedRecords = append(mergedRecords, knownRecords...) + + // Include custom records in the extra data wire field if they are + // present. Ensure that the custom records are validated before encoding + // them. + if err := customRecords.Validate(); err != nil { + return nil, fmt.Errorf("custom records validation error: %w", + err) + } + + // Extend the message extra data records slice with TLV records from the + // custom records field. + mergedRecords = append( + mergedRecords, customRecords.RecordProducers()..., + ) + + // Now we can sort the records and make sure there are no records with + // the same type that would collide when encoding. + sortedRecords := ProduceRecordsSorted(mergedRecords...) + if err := AssertUniqueTypes(sortedRecords); err != nil { + return nil, err + } + + return EncodeRecords(sortedRecords) +} + +// ParseAndExtractExtraData parses the given extra data into the passed-in +// records, then returns any remaining records as extra data. +func ParseAndExtractExtraData(allTlvData ExtraOpaqueData, + knownRecords ...tlv.RecordProducer) (fn.Set[tlv.Type], + ExtraOpaqueData, error) { + + extraDataTlvMap, err := allTlvData.ExtractRecords(knownRecords...) + if err != nil { + return nil, nil, err + } + + // Remove the known and now extracted records from the leftover extra + // data map. + parsedKnownRecords := make(fn.Set[tlv.Type], len(knownRecords)) + for _, producer := range knownRecords { + r := producer.Record() + + // Only remove the records if it was parsed (remainder is nil). + // We'll just store the type so we can tell the caller which + // records were actually parsed fully. + val, ok := extraDataTlvMap[r.Type()] + if ok && val == nil { + parsedKnownRecords.Add(r.Type()) + delete(extraDataTlvMap, r.Type()) + } + } + + // Encode the remaining records back into the extra data field. These + // records are not in the custom records TLV type range and do not + // have associated fields in the struct that produced the records. + extraData, err := NewExtraOpaqueData(extraDataTlvMap) + if err != nil { + return nil, nil, err + } + + return parsedKnownRecords, extraData, nil +} diff --git a/payments/db/migration1/lnwire/features.go b/payments/db/migration1/lnwire/features.go new file mode 100644 index 000000000..4e927e170 --- /dev/null +++ b/payments/db/migration1/lnwire/features.go @@ -0,0 +1,898 @@ +package lnwire + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/lightningnetwork/lnd/tlv" +) + +var ( + // ErrFeaturePairExists signals an error in feature vector construction + // where the opposing bit in a feature pair has already been set. + ErrFeaturePairExists = errors.New("feature pair exists") + + // ErrFeatureStandard is returned when attempts to modify LND's known + // set of features are made. + ErrFeatureStandard = errors.New("feature is used in standard " + + "protocol set") + + // ErrFeatureBitMaximum is returned when a feature bit exceeds the + // maximum allowable value. + ErrFeatureBitMaximum = errors.New("feature bit exceeds allowed maximum") +) + +// FeatureBit represents a feature that can be enabled in either a local or +// global feature vector at a specific bit position. Feature bits follow the +// "it's OK to be odd" rule, where features at even bit positions must be known +// to a node receiving them from a peer while odd bits do not. In accordance, +// feature bits are usually assigned in pairs, first being assigned an odd bit +// position which may later be changed to the preceding even position once +// knowledge of the feature becomes required on the network. +type FeatureBit uint16 + +const ( + // DataLossProtectRequired is a feature bit that indicates that a peer + // *requires* the other party know about the data-loss-protect optional + // feature. If the remote peer does not know of such a feature, then + // the sending peer SHOULD disconnect them. The data-loss-protect + // feature allows a peer that's lost partial data to recover their + // settled funds of the latest commitment state. + DataLossProtectRequired FeatureBit = 0 + + // DataLossProtectOptional is an optional feature bit that indicates + // that the sending peer knows of this new feature and can activate it + // it. The data-loss-protect feature allows a peer that's lost partial + // data to recover their settled funds of the latest commitment state. + DataLossProtectOptional FeatureBit = 1 + + // InitialRoutingSync is a local feature bit meaning that the receiving + // node should send a complete dump of routing information when a new + // connection is established. + InitialRoutingSync FeatureBit = 3 + + // UpfrontShutdownScriptRequired is a feature bit which indicates that a + // peer *requires* that the remote peer accept an upfront shutdown script to + // which payout is enforced on cooperative closes. + UpfrontShutdownScriptRequired FeatureBit = 4 + + // UpfrontShutdownScriptOptional is an optional feature bit which indicates + // that the peer will accept an upfront shutdown script to which payout is + // enforced on cooperative closes. + UpfrontShutdownScriptOptional FeatureBit = 5 + + // GossipQueriesRequired is a feature bit that indicates that the + // receiving peer MUST know of the set of features that allows nodes to + // more efficiently query the network view of peers on the network for + // reconciliation purposes. + GossipQueriesRequired FeatureBit = 6 + + // GossipQueriesOptional is an optional feature bit that signals that + // the setting peer knows of the set of features that allows more + // efficient network view reconciliation. + GossipQueriesOptional FeatureBit = 7 + + // TLVOnionPayloadRequired is a feature bit that indicates a node is + // able to decode the new TLV information included in the onion packet. + TLVOnionPayloadRequired FeatureBit = 8 + + // TLVOnionPayloadOptional is an optional feature bit that indicates a + // node is able to decode the new TLV information included in the onion + // packet. + TLVOnionPayloadOptional FeatureBit = 9 + + // StaticRemoteKeyRequired is a required feature bit that signals that + // within one's commitment transaction, the key used for the remote + // party's non-delay output should not be tweaked. + StaticRemoteKeyRequired FeatureBit = 12 + + // StaticRemoteKeyOptional is an optional feature bit that signals that + // within one's commitment transaction, the key used for the remote + // party's non-delay output should not be tweaked. + StaticRemoteKeyOptional FeatureBit = 13 + + // PaymentAddrRequired is a required feature bit that signals that a + // node requires payment addresses, which are used to mitigate probing + // attacks on the receiver of a payment. + PaymentAddrRequired FeatureBit = 14 + + // PaymentAddrOptional is an optional feature bit that signals that a + // node supports payment addresses, which are used to mitigate probing + // attacks on the receiver of a payment. + PaymentAddrOptional FeatureBit = 15 + + // MPPRequired is a required feature bit that signals that the receiver + // of a payment requires settlement of an invoice with more than one + // HTLC. + MPPRequired FeatureBit = 16 + + // MPPOptional is an optional feature bit that signals that the receiver + // of a payment supports settlement of an invoice with more than one + // HTLC. + MPPOptional FeatureBit = 17 + + // WumboChannelsRequired is a required feature bit that signals that a + // node is willing to accept channels larger than 2^24 satoshis. + WumboChannelsRequired FeatureBit = 18 + + // WumboChannelsOptional is an optional feature bit that signals that a + // node is willing to accept channels larger than 2^24 satoshis. + WumboChannelsOptional FeatureBit = 19 + + // AnchorsRequired is a required feature bit that signals that the node + // requires channels to be made using commitments having anchor + // outputs. + AnchorsRequired FeatureBit = 20 + + // AnchorsOptional is an optional feature bit that signals that the + // node supports channels to be made using commitments having anchor + // outputs. + AnchorsOptional FeatureBit = 21 + + // AnchorsZeroFeeHtlcTxRequired is a required feature bit that signals + // that the node requires channels having zero-fee second-level HTLC + // transactions, which also imply anchor commitments. + AnchorsZeroFeeHtlcTxRequired FeatureBit = 22 + + // AnchorsZeroFeeHtlcTxOptional is an optional feature bit that signals + // that the node supports channels having zero-fee second-level HTLC + // transactions, which also imply anchor commitments. + AnchorsZeroFeeHtlcTxOptional FeatureBit = 23 + + // RouteBlindingRequired is a required feature bit that signals that + // the node supports blinded payments. + RouteBlindingRequired FeatureBit = 24 + + // RouteBlindingOptional is an optional feature bit that signals that + // the node supports blinded payments. + RouteBlindingOptional FeatureBit = 25 + + // ShutdownAnySegwitRequired is an required feature bit that signals + // that the sender is able to properly handle/parse segwit witness + // programs up to version 16. This enables utilization of Taproot + // addresses for cooperative closure addresses. + ShutdownAnySegwitRequired FeatureBit = 26 + + // ShutdownAnySegwitOptional is an optional feature bit that signals + // that the sender is able to properly handle/parse segwit witness + // programs up to version 16. This enables utilization of Taproot + // addresses for cooperative closure addresses. + ShutdownAnySegwitOptional FeatureBit = 27 + + // AMPRequired is a required feature bit that signals that the receiver + // of a payment supports accepts spontaneous payments, i.e. + // sender-generated preimages according to BOLT XX. + AMPRequired FeatureBit = 30 + + // AMPOptional is an optional feature bit that signals that the receiver + // of a payment supports accepts spontaneous payments, i.e. + // sender-generated preimages according to BOLT XX. + AMPOptional FeatureBit = 31 + + // QuiescenceRequired is a required feature bit that denotes that a + // connection established with this node must support the quiescence + // protocol if it wants to have a channel relationship. + QuiescenceRequired FeatureBit = 34 + + // QuiescenceOptional is an optional feature bit that denotes that a + // connection established with this node is permitted to use the + // quiescence protocol. + QuiescenceOptional FeatureBit = 35 + + // ExplicitChannelTypeRequired is a required bit that denotes that a + // connection established with this node is to use explicit channel + // commitment types for negotiation instead of the existing implicit + // negotiation methods. With this bit, there is no longer a "default" + // implicit channel commitment type, allowing a connection to + // open/maintain types of several channels over its lifetime. + ExplicitChannelTypeRequired = 44 + + // ExplicitChannelTypeOptional is an optional bit that denotes that a + // connection established with this node is to use explicit channel + // commitment types for negotiation instead of the existing implicit + // negotiation methods. With this bit, there is no longer a "default" + // implicit channel commitment type, allowing a connection to + // TODO: Decide on actual feature bit value. + ExplicitChannelTypeOptional = 45 + + // ScidAliasRequired is a required feature bit that signals that the + // node requires understanding of ShortChannelID aliases in the TLV + // segment of the channel_ready message. + ScidAliasRequired FeatureBit = 46 + + // ScidAliasOptional is an optional feature bit that signals that the + // node understands ShortChannelID aliases in the TLV segment of the + // channel_ready message. + ScidAliasOptional FeatureBit = 47 + + // PaymentMetadataRequired is a required bit that denotes that if an + // invoice contains metadata, it must be passed along with the payment + // htlc(s). + PaymentMetadataRequired = 48 + + // PaymentMetadataOptional is an optional bit that denotes that if an + // invoice contains metadata, it may be passed along with the payment + // htlc(s). + PaymentMetadataOptional = 49 + + // ZeroConfRequired is a required feature bit that signals that the + // node requires understanding of the zero-conf channel_type. + ZeroConfRequired FeatureBit = 50 + + // ZeroConfOptional is an optional feature bit that signals that the + // node understands the zero-conf channel type. + ZeroConfOptional FeatureBit = 51 + + // KeysendRequired is a required bit that indicates that the node is + // able and willing to accept keysend payments. + KeysendRequired = 54 + + // KeysendOptional is an optional bit that indicates that the node is + // able and willing to accept keysend payments. + KeysendOptional = 55 + + // RbfCoopCloseRequired is a required feature bit that signals that + // the new RBF-based co-op close protocol is supported. + RbfCoopCloseRequired = 60 + + // RbfCoopCloseOptional is an optional feature bit that signals that the + // new RBF-based co-op close protocol is supported. + RbfCoopCloseOptional = 61 + + // RbfCoopCloseRequiredStaging is a required feature bit that signals + // that the new RBF-based co-op close protocol is supported. + RbfCoopCloseRequiredStaging = 160 + + // RbfCoopCloseOptionalStaging is an optional feature bit that signals + // that the new RBF-based co-op close protocol is supported. + RbfCoopCloseOptionalStaging = 161 + + // ScriptEnforcedLeaseRequired is a required feature bit that signals + // that the node requires channels having zero-fee second-level HTLC + // transactions, which also imply anchor commitments, along with an + // additional CLTV constraint of a channel lease's expiration height + // applied to all outputs that pay directly to the channel initiator. + // + // TODO: Decide on actual feature bit value. + ScriptEnforcedLeaseRequired FeatureBit = 2022 + + // ScriptEnforcedLeaseOptional is an optional feature bit that signals + // that the node requires channels having zero-fee second-level HTLC + // transactions, which also imply anchor commitments, along with an + // additional CLTV constraint of a channel lease's expiration height + // applied to all outputs that pay directly to the channel initiator. + // + // TODO: Decide on actual feature bit value. + ScriptEnforcedLeaseOptional FeatureBit = 2023 + + // SimpleTaprootChannelsRequiredFinal is a required bit that indicates + // the node is able to create taproot-native channels. This is the + // final feature bit to be used once the channel type is finalized. + SimpleTaprootChannelsRequiredFinal = 80 + + // SimpleTaprootChannelsOptionalFinal is an optional bit that indicates + // the node is able to create taproot-native channels. This is the + // final feature bit to be used once the channel type is finalized. + SimpleTaprootChannelsOptionalFinal = 81 + + // SimpleTaprootChannelsRequiredStaging is a required bit that indicates + // the node is able to create taproot-native channels. This is a + // feature bit used in the wild while the channel type is still being + // finalized. + SimpleTaprootChannelsRequiredStaging = 180 + + // SimpleTaprootChannelsOptionalStaging is an optional bit that + // indicates the node is able to create taproot-native channels. This + // is a feature bit used in the wild while the channel type is still + // being finalized. + SimpleTaprootChannelsOptionalStaging = 181 + + // ExperimentalAccountabilityRequired is a required feature bit that + // indicates that the node will relay experimental accountability + // signals. + ExperimentalAccountabilityRequired FeatureBit = 260 + + // ExperimentalAccountabilityOptional is an optional feature bit that + // indicates that the node will relay experimental accountability + // signals. + ExperimentalAccountabilityOptional FeatureBit = 261 + + // Bolt11BlindedPathsRequired is a required feature bit that indicates + // that the node is able to understand the blinded path tagged field in + // a BOLT 11 invoice. + Bolt11BlindedPathsRequired = 262 + + // Bolt11BlindedPathsOptional is an optional feature bit that indicates + // that the node is able to understand the blinded path tagged field in + // a BOLT 11 invoice. + Bolt11BlindedPathsOptional = 263 + + // SimpleTaprootOverlayChansRequired is a required bit that indicates + // support for the special custom taproot overlay channel. + SimpleTaprootOverlayChansOptional = 2025 + + // SimpleTaprootOverlayChansRequired is a required bit that indicates + // support for the special custom taproot overlay channel. + SimpleTaprootOverlayChansRequired = 2026 + + // MaxBolt11Feature is the maximum feature bit value allowed in bolt 11 + // invoices. + // + // The base 32 encoded tagged fields in invoices are limited to 10 bits + // to express the length of the field's data. + //nolint:ll + // See: https://github.com/lightning/bolts/blob/master/11-payment-encoding.md#tagged-fields + // + // With a maximum length field of 1023 (2^10 -1) and 5 bit encoding, + // the highest feature bit that can be expressed is: + // 1023 * 5 - 1 = 5114. + MaxBolt11Feature = 5114 +) + +// IsRequired returns true if the feature bit is even, and false otherwise. +func (b FeatureBit) IsRequired() bool { + return b&0x01 == 0x00 +} + +// Features is a mapping of known feature bits to a descriptive name. All known +// feature bits must be assigned a name in this mapping, and feature bit pairs +// must be assigned together for correct behavior. +var Features = map[FeatureBit]string{ + DataLossProtectRequired: "data-loss-protect", + DataLossProtectOptional: "data-loss-protect", + InitialRoutingSync: "initial-routing-sync", + UpfrontShutdownScriptRequired: "upfront-shutdown-script", + UpfrontShutdownScriptOptional: "upfront-shutdown-script", + GossipQueriesRequired: "gossip-queries", + GossipQueriesOptional: "gossip-queries", + TLVOnionPayloadRequired: "tlv-onion", + TLVOnionPayloadOptional: "tlv-onion", + StaticRemoteKeyOptional: "static-remote-key", + StaticRemoteKeyRequired: "static-remote-key", + PaymentAddrOptional: "payment-addr", + PaymentAddrRequired: "payment-addr", + MPPOptional: "multi-path-payments", + MPPRequired: "multi-path-payments", + AnchorsRequired: "anchor-commitments", + AnchorsOptional: "anchor-commitments", + AnchorsZeroFeeHtlcTxRequired: "anchors-zero-fee-htlc-tx", + AnchorsZeroFeeHtlcTxOptional: "anchors-zero-fee-htlc-tx", + WumboChannelsRequired: "wumbo-channels", + WumboChannelsOptional: "wumbo-channels", + AMPRequired: "amp", + AMPOptional: "amp", + QuiescenceRequired: "quiescence", + QuiescenceOptional: "quiescence", + PaymentMetadataOptional: "payment-metadata", + PaymentMetadataRequired: "payment-metadata", + ExplicitChannelTypeOptional: "explicit-commitment-type", + ExplicitChannelTypeRequired: "explicit-commitment-type", + KeysendOptional: "keysend", + KeysendRequired: "keysend", + ScriptEnforcedLeaseRequired: "script-enforced-lease", + ScriptEnforcedLeaseOptional: "script-enforced-lease", + ScidAliasRequired: "scid-alias", + ScidAliasOptional: "scid-alias", + ZeroConfRequired: "zero-conf", + ZeroConfOptional: "zero-conf", + RouteBlindingRequired: "route-blinding", + RouteBlindingOptional: "route-blinding", + ShutdownAnySegwitRequired: "shutdown-any-segwit", + ShutdownAnySegwitOptional: "shutdown-any-segwit", + SimpleTaprootChannelsRequiredFinal: "simple-taproot-chans", + SimpleTaprootChannelsOptionalFinal: "simple-taproot-chans", + SimpleTaprootChannelsRequiredStaging: "simple-taproot-chans-x", + SimpleTaprootChannelsOptionalStaging: "simple-taproot-chans-x", + SimpleTaprootOverlayChansOptional: "taproot-overlay-chans", + SimpleTaprootOverlayChansRequired: "taproot-overlay-chans", + ExperimentalAccountabilityRequired: "accountable-x", + ExperimentalAccountabilityOptional: "accountable-x", + Bolt11BlindedPathsOptional: "bolt-11-blinded-paths", + Bolt11BlindedPathsRequired: "bolt-11-blinded-paths", + RbfCoopCloseOptional: "rbf-coop-close", + RbfCoopCloseRequired: "rbf-coop-close", + RbfCoopCloseOptionalStaging: "rbf-coop-close-x", + RbfCoopCloseRequiredStaging: "rbf-coop-close-x", +} + +// RawFeatureVector represents a set of feature bits as defined in BOLT-09. A +// RawFeatureVector itself just stores a set of bit flags but can be used to +// construct a FeatureVector which binds meaning to each bit. Feature vectors +// can be serialized and deserialized to/from a byte representation that is +// transmitted in Lightning network messages. +type RawFeatureVector struct { + features map[FeatureBit]struct{} +} + +// NewRawFeatureVector creates a feature vector with all of the feature bits +// given as arguments enabled. +func NewRawFeatureVector(bits ...FeatureBit) *RawFeatureVector { + fv := &RawFeatureVector{features: make(map[FeatureBit]struct{})} + for _, bit := range bits { + fv.Set(bit) + } + return fv +} + +// IsEmpty returns whether the feature vector contains any feature bits. +func (fv RawFeatureVector) IsEmpty() bool { + return len(fv.features) == 0 +} + +// OnlyContains determines whether only the specified feature bits are found. +func (fv RawFeatureVector) OnlyContains(bits ...FeatureBit) bool { + if len(bits) != len(fv.features) { + return false + } + for _, bit := range bits { + if !fv.IsSet(bit) { + return false + } + } + return true +} + +// Equals determines whether two features vectors contain exactly the same +// features. +func (fv RawFeatureVector) Equals(other *RawFeatureVector) bool { + if len(fv.features) != len(other.features) { + return false + } + for bit := range fv.features { + if _, ok := other.features[bit]; !ok { + return false + } + } + return true +} + +// Merge sets all feature bits in other on the receiver's feature vector. +func (fv *RawFeatureVector) Merge(other *RawFeatureVector) error { + for bit := range other.features { + err := fv.SafeSet(bit) + if err != nil { + return err + } + } + return nil +} + +// ValidateUpdate checks whether a feature vector can safely be updated to the +// new feature vector provided, checking that it does not alter any of the +// "standard" features that are defined by LND. The new feature vector should +// be inclusive of all features in the original vector that it still wants to +// advertise, setting and unsetting updates as desired. Features in the vector +// are also checked against a maximum inclusive value, as feature vectors in +// different contexts have different maximum values. +func (fv *RawFeatureVector) ValidateUpdate(other *RawFeatureVector, + maximumValue FeatureBit) error { + + // Run through the new set of features and check that we're not adding + // any feature bits that are defined but not set in LND. + for feature := range other.features { + if fv.IsSet(feature) { + continue + } + + if feature > maximumValue { + return fmt.Errorf("can't set feature bit %d: %w %v", + feature, ErrFeatureBitMaximum, + maximumValue) + } + + if name, known := Features[feature]; known { + return fmt.Errorf("can't set feature "+ + "bit %d (%v): %w", feature, name, + ErrFeatureStandard) + } + } + + // Check that the new feature vector for this set does not unset any + // features that are standard in LND by comparing the features in our + // current set to the omitted values in the new set. + for feature := range fv.features { + if other.IsSet(feature) { + continue + } + + if name, known := Features[feature]; known { + return fmt.Errorf("can't unset feature "+ + "bit %d (%v): %w", feature, name, + ErrFeatureStandard) + } + } + + return nil +} + +// ValidatePairs checks each feature bit in a raw vector to ensure that the +// opposing bit is not set, validating that the vector has either the optional +// or required bit set, not both. +func (fv *RawFeatureVector) ValidatePairs() error { + for feature := range fv.features { + if _, ok := fv.features[feature^1]; ok { + return ErrFeaturePairExists + } + } + + return nil +} + +// Clone makes a copy of a feature vector. +func (fv *RawFeatureVector) Clone() *RawFeatureVector { + newFeatures := NewRawFeatureVector() + for bit := range fv.features { + newFeatures.Set(bit) + } + return newFeatures +} + +// IsSet returns whether a particular feature bit is enabled in the vector. +func (fv *RawFeatureVector) IsSet(feature FeatureBit) bool { + _, ok := fv.features[feature] + return ok +} + +// Set marks a feature as enabled in the vector. +func (fv *RawFeatureVector) Set(feature FeatureBit) { + fv.features[feature] = struct{}{} +} + +// SafeSet sets the chosen feature bit in the feature vector, but returns an +// error if the opposing feature bit is already set. This ensures both that we +// are creating properly structured feature vectors, and in some cases, that +// peers are sending properly encoded ones, i.e. it can't be both optional and +// required. +func (fv *RawFeatureVector) SafeSet(feature FeatureBit) error { + if _, ok := fv.features[feature^1]; ok { + return ErrFeaturePairExists + } + + fv.Set(feature) + return nil +} + +// Unset marks a feature as disabled in the vector. +func (fv *RawFeatureVector) Unset(feature FeatureBit) { + delete(fv.features, feature) +} + +// SerializeSize returns the number of bytes needed to represent feature vector +// in byte format. +func (fv *RawFeatureVector) SerializeSize() int { + // We calculate byte-length via the largest bit index. + return fv.serializeSize(8) +} + +// SerializeSize32 returns the number of bytes needed to represent feature +// vector in base32 format. +func (fv *RawFeatureVector) SerializeSize32() int { + // We calculate base32-length via the largest bit index. + return fv.serializeSize(5) +} + +// serializeSize returns the number of bytes required to encode the feature +// vector using at most width bits per encoded byte. +func (fv *RawFeatureVector) serializeSize(width int) int { + // Find the largest feature bit index + max := -1 + for feature := range fv.features { + index := int(feature) + if index > max { + max = index + } + } + if max == -1 { + return 0 + } + + return max/width + 1 +} + +// Encode writes the feature vector in byte representation. Every feature +// encoded as a bit, and the bit vector is serialized using the least number of +// bytes. Since the bit vector length is variable, the first two bytes of the +// serialization represent the length. +func (fv *RawFeatureVector) Encode(w io.Writer) error { + // Write length of feature vector. + var l [2]byte + length := fv.SerializeSize() + binary.BigEndian.PutUint16(l[:], uint16(length)) + if _, err := w.Write(l[:]); err != nil { + return err + } + + return fv.encode(w, length, 8) +} + +// EncodeBase256 writes the feature vector in base256 representation. Every +// feature is encoded as a bit, and the bit vector is serialized using the least +// number of bytes. +func (fv *RawFeatureVector) EncodeBase256(w io.Writer) error { + length := fv.SerializeSize() + return fv.encode(w, length, 8) +} + +// EncodeBase32 writes the feature vector in base32 representation. Every feature +// is encoded as a bit, and the bit vector is serialized using the least number of +// bytes. +func (fv *RawFeatureVector) EncodeBase32(w io.Writer) error { + length := fv.SerializeSize32() + return fv.encode(w, length, 5) +} + +// encode writes the feature vector +func (fv *RawFeatureVector) encode(w io.Writer, length, width int) error { + // Generate the data and write it. + data := make([]byte, length) + for feature := range fv.features { + byteIndex := int(feature) / width + bitIndex := int(feature) % width + data[length-byteIndex-1] |= 1 << uint(bitIndex) + } + + _, err := w.Write(data) + return err +} + +// Decode reads the feature vector from its byte representation. Every feature +// is encoded as a bit, and the bit vector is serialized using the least number +// of bytes. Since the bit vector length is variable, the first two bytes of the +// serialization represent the length. +func (fv *RawFeatureVector) Decode(r io.Reader) error { + // Read the length of the feature vector. + var l [2]byte + if _, err := io.ReadFull(r, l[:]); err != nil { + return err + } + length := binary.BigEndian.Uint16(l[:]) + + return fv.decode(r, int(length), 8) +} + +// DecodeBase256 reads the feature vector from its base256 representation. Every +// feature encoded as a bit, and the bit vector is serialized using the least +// number of bytes. +func (fv *RawFeatureVector) DecodeBase256(r io.Reader, length int) error { + return fv.decode(r, length, 8) +} + +// DecodeBase32 reads the feature vector from its base32 representation. Every +// feature encoded as a bit, and the bit vector is serialized using the least +// number of bytes. +func (fv *RawFeatureVector) DecodeBase32(r io.Reader, length int) error { + return fv.decode(r, length, 5) +} + +// decode reads a feature vector from the next length bytes of the io.Reader, +// assuming each byte has width feature bits encoded per byte. +func (fv *RawFeatureVector) decode(r io.Reader, length, width int) error { + // Read the feature vector data. + data := make([]byte, length) + if _, err := io.ReadFull(r, data); err != nil { + return err + } + + // Set feature bits from parsed data. + bitsNumber := len(data) * width + for i := 0; i < bitsNumber; i++ { + byteIndex := int(i / width) + bitIndex := uint(i % width) + if (data[length-byteIndex-1]>>bitIndex)&1 == 1 { + fv.Set(FeatureBit(i)) + } + } + + return nil +} + +// sizeFunc returns the length required to encode the feature vector. +func (fv *RawFeatureVector) sizeFunc() uint64 { + return uint64(fv.SerializeSize()) +} + +// Record returns a TLV record that can be used to encode/decode raw feature +// vectors. Note that the length of the feature vector is not included, because +// it is covered by the TLV record's length field. +func (fv *RawFeatureVector) Record() tlv.Record { + return tlv.MakeDynamicRecord( + 0, fv, fv.sizeFunc, rawFeatureEncoder, rawFeatureDecoder, + ) +} + +// rawFeatureEncoder is a custom TLV encoder for raw feature vectors. +func rawFeatureEncoder(w io.Writer, val interface{}, _ *[8]byte) error { + if v, ok := val.(*RawFeatureVector); ok { + // Encode the feature bits as a byte slice without its length + // prepended, as that's already taken care of by the TLV record. + fv := *v + return fv.encode(w, fv.SerializeSize(), 8) + } + + return tlv.NewTypeForEncodingErr(val, "lnwire.RawFeatureVector") +} + +// rawFeatureDecoder is a custom TLV decoder for raw feature vectors. +func rawFeatureDecoder(r io.Reader, val interface{}, _ *[8]byte, + l uint64) error { + + if v, ok := val.(*RawFeatureVector); ok { + fv := NewRawFeatureVector() + if err := fv.decode(r, int(l), 8); err != nil { + return err + } + *v = *fv + + return nil + } + + return tlv.NewTypeForEncodingErr(val, "lnwire.RawFeatureVector") +} + +// FeatureVector represents a set of enabled features. The set stores +// information on enabled flags and metadata about the feature names. A feature +// vector is serializable to a compact byte representation that is included in +// Lightning network messages. +type FeatureVector struct { + *RawFeatureVector + featureNames map[FeatureBit]string +} + +// NewFeatureVector constructs a new FeatureVector from a raw feature vector +// and mapping of feature definitions. If the feature vector argument is nil, a +// new one will be constructed with no enabled features. +func NewFeatureVector(featureVector *RawFeatureVector, + featureNames map[FeatureBit]string) *FeatureVector { + + if featureVector == nil { + featureVector = NewRawFeatureVector() + } + return &FeatureVector{ + RawFeatureVector: featureVector, + featureNames: featureNames, + } +} + +// EmptyFeatureVector returns a feature vector with no bits set. +func EmptyFeatureVector() *FeatureVector { + return NewFeatureVector(nil, Features) +} + +// Record implements the RecordProducer interface for FeatureVector. Note that +// it uses a zero-value type is used to produce the record, as we expect this +// type value to be overwritten when used in generic TLV record production. +// This allows a single Record function to serve in the many different contexts +// in which feature vectors are encoded. This record wraps the encoding/ +// decoding for our raw feature vectors so that we can directly parse fully +// formed feature vector types. +func (fv *FeatureVector) Record() tlv.Record { + return tlv.MakeDynamicRecord(0, fv, fv.sizeFunc, + func(w io.Writer, val interface{}, buf *[8]byte) error { + if f, ok := val.(*FeatureVector); ok { + return rawFeatureEncoder( + w, f.RawFeatureVector, buf, + ) + } + + return tlv.NewTypeForEncodingErr( + val, "*lnwire.FeatureVector", + ) + }, + func(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if f, ok := val.(*FeatureVector); ok { + features := NewFeatureVector(nil, Features) + err := rawFeatureDecoder( + r, features.RawFeatureVector, buf, l, + ) + if err != nil { + return err + } + + *f = *features + + return nil + } + + return tlv.NewTypeForDecodingErr( + val, "*lnwire.FeatureVector", l, l, + ) + }, + ) +} + +// HasFeature returns whether a particular feature is included in the set. The +// feature can be seen as set either if the bit is set directly OR the queried +// bit has the same meaning as its corresponding even/odd bit, which is set +// instead. The second case is because feature bits are generally assigned in +// pairs where both the even and odd position represent the same feature. +func (fv *FeatureVector) HasFeature(feature FeatureBit) bool { + return fv.IsSet(feature) || + (fv.isFeatureBitPair(feature) && fv.IsSet(feature^1)) +} + +// RequiresFeature returns true if the referenced feature vector *requires* +// that the given required bit be set. This method can be used with both +// optional and required feature bits as a parameter. +func (fv *FeatureVector) RequiresFeature(feature FeatureBit) bool { + // If we weren't passed a required feature bit, then we'll flip the + // lowest bit to query for the required version of the feature. This + // lets callers pass in both the optional and required bits. + if !feature.IsRequired() { + feature ^= 1 + } + + return fv.IsSet(feature) +} + +// UnknownRequiredFeatures returns a list of feature bits set in the vector +// that are unknown and in an even bit position. Feature bits with an even +// index must be known to a node receiving the feature vector in a message. +func (fv *FeatureVector) UnknownRequiredFeatures() []FeatureBit { + var unknown []FeatureBit + for feature := range fv.features { + if feature%2 == 0 && !fv.IsKnown(feature) { + unknown = append(unknown, feature) + } + } + return unknown +} + +// UnknownFeatures returns a boolean if a feature vector contains *any* +// unknown features (even if they are odd). +func (fv *FeatureVector) UnknownFeatures() bool { + for feature := range fv.features { + if !fv.IsKnown(feature) { + return true + } + } + + return false +} + +// Name returns a string identifier for the feature represented by this bit. If +// the bit does not represent a known feature, this returns a string indicating +// as such. +func (fv *FeatureVector) Name(bit FeatureBit) string { + name, known := fv.featureNames[bit] + if !known { + return "unknown" + } + return name +} + +// IsKnown returns whether this feature bit represents a known feature. +func (fv *FeatureVector) IsKnown(bit FeatureBit) bool { + _, known := fv.featureNames[bit] + return known +} + +// isFeatureBitPair returns whether this feature bit and its corresponding +// even/odd bit both represent the same feature. This may often be the case as +// bits are generally assigned in pairs, first being assigned an odd bit +// position then being promoted to an even bit position once the network is +// ready. +func (fv *FeatureVector) isFeatureBitPair(bit FeatureBit) bool { + name1, known1 := fv.featureNames[bit] + name2, known2 := fv.featureNames[bit^1] + return known1 && known2 && name1 == name2 +} + +// Features returns the set of raw features contained in the feature vector. +func (fv *FeatureVector) Features() map[FeatureBit]struct{} { + fs := make(map[FeatureBit]struct{}, len(fv.RawFeatureVector.features)) + for b := range fv.RawFeatureVector.features { + fs[b] = struct{}{} + } + return fs +} + +// Clone copies a feature vector, carrying over its feature bits. The feature +// names are not copied. +func (fv *FeatureVector) Clone() *FeatureVector { + features := fv.RawFeatureVector.Clone() + return NewFeatureVector(features, fv.featureNames) +} diff --git a/payments/db/migration1/lnwire/interfaces.go b/payments/db/migration1/lnwire/interfaces.go new file mode 100644 index 000000000..115b07d7b --- /dev/null +++ b/payments/db/migration1/lnwire/interfaces.go @@ -0,0 +1,174 @@ +package lnwire + +import ( + "fmt" + + "github.com/btcsuite/btcd/chaincfg/chainhash" +) + +// GossipVersion is a version number that describes the version of the +// gossip protocol that a gossip message was gossiped on. +type GossipVersion uint8 + +const ( + // GossipVersion1 is the initial version of the gossip protocol as + // defined in BOLT 7. This version of the protocol can only gossip P2WSH + // channels and makes use of ECDSA signatures. + GossipVersion1 GossipVersion = 1 + + // GossipVersion2 is the newest version of the gossip protocol. This + // version adds support for P2TR channels and makes use of Schnorr + // signatures. The BOLT number is TBD. + GossipVersion2 GossipVersion = 2 +) + +// String returns a string representation of the protocol version. +func (v GossipVersion) String() string { + return fmt.Sprintf("V%d", v) +} + +// GossipMessage is an interface that must be satisfied by all messages that are +// part of the gossip protocol. +type GossipMessage interface { + // GossipVersion returns the version of the gossip protocol that a + // message is part of. + GossipVersion() GossipVersion +} + +// AnnounceSignatures is an interface that represents a message used to +// exchange signatures of a ChannelAnnouncment message during the funding flow. +type AnnounceSignatures interface { + // SCID returns the ShortChannelID of the channel. + SCID() ShortChannelID + + // ChanID returns the ChannelID identifying the channel. + ChanID() ChannelID + + Message + GossipMessage +} + +// ChannelAnnouncement is an interface that must be satisfied by any message +// used to announce and prove the existence of a channel. +type ChannelAnnouncement interface { + // SCID returns the short channel ID of the channel. + SCID() ShortChannelID + + // GetChainHash returns the hash of the chain which this channel's + // funding transaction is confirmed in. + GetChainHash() chainhash.Hash + + // Node1KeyBytes returns the bytes representing the public key of node + // 1 in the channel. + Node1KeyBytes() [33]byte + + // Node2KeyBytes returns the bytes representing the public key of node + // 2 in the channel. + Node2KeyBytes() [33]byte + + Message + GossipMessage +} + +// CompareResult represents the result after comparing two things. +type CompareResult uint8 + +const ( + // LessThan indicates that base object is less than the object it was + // compared to. + LessThan CompareResult = iota + + // EqualTo indicates that the base object is equal to the object it was + // compared to. + EqualTo + + // GreaterThan indicates that base object is greater than the object it + // was compared to. + GreaterThan +) + +// ChannelUpdate is an interface that describes a message used to update the +// forwarding rules of a channel. +type ChannelUpdate interface { + // SCID returns the ShortChannelID of the channel that the update + // applies to. + SCID() ShortChannelID + + // IsNode1 is true if the update was produced by node 1 of the channel + // peers. Node 1 is the node with the lexicographically smaller public + // key. + IsNode1() bool + + // IsDisabled is true if the update is announcing that the channel + // should be considered disabled. + IsDisabled() bool + + // GetChainHash returns the hash of the chain that the message is + // referring to. + GetChainHash() chainhash.Hash + + // ForwardingPolicy returns the set of forwarding constraints of the + // update. + ForwardingPolicy() *ForwardingPolicy + + // CmpAge can be used to determine if the update is older or newer than + // the passed update. It returns LessThan if this update is older than + // the passed update, GreaterThan if it is newer and EqualTo if they are + // the same age. + CmpAge(update ChannelUpdate) (CompareResult, error) + + // SetDisabledFlag can be used to adjust the disabled flag of an update. + SetDisabledFlag(bool) + + // SetSCID can be used to overwrite the SCID of the update. + SetSCID(scid ShortChannelID) + + Message + GossipMessage +} + +// NodeAnnouncement is an interface that must be satisfied by any message used +// to announce the existence of a node. +type NodeAnnouncement interface { + // NodePub returns the identity public key of the node. + NodePub() [33]byte + + // NodeFeatures returns the set of features supported by the node. + NodeFeatures() *FeatureVector + + // TimestampDesc returns a human-readable description of the + // timestamp of the announcement. + TimestampDesc() string + + Message + GossipMessage +} + +// ForwardingPolicy defines the set of forwarding constraints advertised in a +// ChannelUpdate message. +type ForwardingPolicy struct { + // TimeLockDelta is the minimum number of blocks that the node requires + // to be added to the expiry of HTLCs. This is a security parameter + // determined by the node operator. This value represents the required + // gap between the time locks of the incoming and outgoing HTLC's set + // to this node. + TimeLockDelta uint16 + + // BaseFee is the base fee that must be used for incoming HTLC's to + // this particular channel. This value will be tacked onto the required + // for a payment independent of the size of the payment. + BaseFee MilliSatoshi + + // FeeRate is the fee rate that will be charged per millionth of a + // satoshi. + FeeRate MilliSatoshi + + // HtlcMinimumMsat is the minimum HTLC value which will be accepted. + MinHTLC MilliSatoshi + + // HasMaxHTLC is true if the MaxHTLC field is provided in the update. + HasMaxHTLC bool + + // HtlcMaximumMsat is the maximum HTLC value which will be accepted. + MaxHTLC MilliSatoshi +} diff --git a/payments/db/migration1/lnwire/lnwire.go b/payments/db/migration1/lnwire/lnwire.go new file mode 100644 index 000000000..09057dae4 --- /dev/null +++ b/payments/db/migration1/lnwire/lnwire.go @@ -0,0 +1,426 @@ +package lnwire + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/btcutil" +) + +const ( + // MaxSliceLength is the maximum allowed length for any opaque byte + // slices in the wire protocol. + MaxSliceLength = 65535 + + // MaxMsgBody is the largest payload any message is allowed to provide. + // This is two less than the MaxSliceLength as each message has a 2 + // byte type that precedes the message body. + MaxMsgBody = 65533 +) + +// PkScript is simple type definition which represents a raw serialized public +// key script. +type PkScript []byte + +// WriteElement is a one-stop shop to write the big endian representation of +// any element which is to be serialized for the wire protocol. +// +// TODO(yy): rm this method once we finish dereferencing it from other +// packages. +func WriteElement(w *bytes.Buffer, element interface{}) error { + switch e := element.(type) { + case uint8: + var b [1]byte + b[0] = e + if _, err := w.Write(b[:]); err != nil { + return err + } + + case uint16: + var b [2]byte + binary.BigEndian.PutUint16(b[:], e) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case ChanUpdateMsgFlags: + var b [1]byte + b[0] = uint8(e) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case ChanUpdateChanFlags: + var b [1]byte + b[0] = uint8(e) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case MilliSatoshi: + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(e)) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case btcutil.Amount: + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(e)) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case uint32: + var b [4]byte + binary.BigEndian.PutUint32(b[:], e) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case uint64: + var b [8]byte + binary.BigEndian.PutUint64(b[:], e) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case *btcec.PublicKey: + if e == nil { + return fmt.Errorf("cannot write nil pubkey") + } + + var b [33]byte + serializedPubkey := e.SerializeCompressed() + copy(b[:], serializedPubkey) + if _, err := w.Write(b[:]); err != nil { + return err + } + + case []Sig: + var b [2]byte + numSigs := uint16(len(e)) + binary.BigEndian.PutUint16(b[:], numSigs) + if _, err := w.Write(b[:]); err != nil { + return err + } + + for _, sig := range e { + if err := WriteElement(w, sig); err != nil { + return err + } + } + + case Sig: + // Write buffer + if _, err := w.Write(e.bytes[:]); err != nil { + return err + } + + case ErrorData: + var l [2]byte + binary.BigEndian.PutUint16(l[:], uint16(len(e))) + if _, err := w.Write(l[:]); err != nil { + return err + } + + if _, err := w.Write(e[:]); err != nil { + return err + } + + case [33]byte: + if _, err := w.Write(e[:]); err != nil { + return err + } + + case []byte: + if _, err := w.Write(e[:]); err != nil { + return err + } + + case *RawFeatureVector: + if e == nil { + return fmt.Errorf("cannot write nil feature vector") + } + + if err := e.Encode(w); err != nil { + return err + } + + case ChannelID: + if _, err := w.Write(e[:]); err != nil { + return err + } + + case FailCode: + if err := WriteElement(w, uint16(e)); err != nil { + return err + } + + case ShortChannelID: + // Check that field fit in 3 bytes and write the blockHeight + if e.BlockHeight > ((1 << 24) - 1) { + return errors.New("block height should fit in 3 bytes") + } + + var blockHeight [4]byte + binary.BigEndian.PutUint32(blockHeight[:], e.BlockHeight) + + if _, err := w.Write(blockHeight[1:]); err != nil { + return err + } + + // Check that field fit in 3 bytes and write the txIndex + if e.TxIndex > ((1 << 24) - 1) { + return errors.New("tx index should fit in 3 bytes") + } + + var txIndex [4]byte + binary.BigEndian.PutUint32(txIndex[:], e.TxIndex) + if _, err := w.Write(txIndex[1:]); err != nil { + return err + } + + // Write the txPosition + var txPosition [2]byte + binary.BigEndian.PutUint16(txPosition[:], e.TxPosition) + if _, err := w.Write(txPosition[:]); err != nil { + return err + } + + case bool: + var b [1]byte + if e { + b[0] = 1 + } + if _, err := w.Write(b[:]); err != nil { + return err + } + + case ExtraOpaqueData: + return e.Encode(w) + + default: + return fmt.Errorf("unknown type in WriteElement: %T", e) + } + + return nil +} + +// WriteElements is writes each element in the elements slice to the passed +// buffer using WriteElement. +// +// TODO(yy): rm this method once we finish dereferencing it from other +// packages. +func WriteElements(buf *bytes.Buffer, elements ...interface{}) error { + for _, element := range elements { + err := WriteElement(buf, element) + if err != nil { + return err + } + } + return nil +} + +// ReadElement is a one-stop utility function to deserialize any datastructure +// encoded using the serialization format of lnwire. +func ReadElement(r io.Reader, element interface{}) error { + var err error + switch e := element.(type) { + case *bool: + var b [1]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + + if b[0] == 1 { + *e = true + } + + case *uint8: + var b [1]uint8 + if _, err := r.Read(b[:]); err != nil { + return err + } + *e = b[0] + + case *uint16: + var b [2]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + *e = binary.BigEndian.Uint16(b[:]) + + case *ChanUpdateMsgFlags: + var b [1]uint8 + if _, err := r.Read(b[:]); err != nil { + return err + } + *e = ChanUpdateMsgFlags(b[0]) + + case *ChanUpdateChanFlags: + var b [1]uint8 + if _, err := r.Read(b[:]); err != nil { + return err + } + *e = ChanUpdateChanFlags(b[0]) + + case *uint32: + var b [4]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + *e = binary.BigEndian.Uint32(b[:]) + + case *uint64: + var b [8]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + *e = binary.BigEndian.Uint64(b[:]) + + case *MilliSatoshi: + var b [8]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + *e = MilliSatoshi(int64(binary.BigEndian.Uint64(b[:]))) + + case *btcutil.Amount: + var b [8]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return err + } + *e = btcutil.Amount(int64(binary.BigEndian.Uint64(b[:]))) + + case **btcec.PublicKey: + var b [btcec.PubKeyBytesLenCompressed]byte + if _, err = io.ReadFull(r, b[:]); err != nil { + return err + } + + pubKey, err := btcec.ParsePubKey(b[:]) + if err != nil { + return err + } + *e = pubKey + + case *RawFeatureVector: + f := NewRawFeatureVector() + err = f.Decode(r) + if err != nil { + return err + } + *e = *f + + case **RawFeatureVector: + f := NewRawFeatureVector() + err = f.Decode(r) + if err != nil { + return err + } + *e = f + + case *[]Sig: + var l [2]byte + if _, err := io.ReadFull(r, l[:]); err != nil { + return err + } + numSigs := binary.BigEndian.Uint16(l[:]) + + var sigs []Sig + if numSigs > 0 { + sigs = make([]Sig, numSigs) + for i := 0; i < int(numSigs); i++ { + if err := ReadElement(r, &sigs[i]); err != nil { + return err + } + } + } + *e = sigs + + case *Sig: + if _, err := io.ReadFull(r, e.bytes[:]); err != nil { + return err + } + + case *ErrorData: + var l [2]byte + if _, err := io.ReadFull(r, l[:]); err != nil { + return err + } + errorLen := binary.BigEndian.Uint16(l[:]) + + *e = ErrorData(make([]byte, errorLen)) + if _, err := io.ReadFull(r, *e); err != nil { + return err + } + + case *[33]byte: + if _, err := io.ReadFull(r, e[:]); err != nil { + return err + } + + case []byte: + if _, err := io.ReadFull(r, e); err != nil { + return err + } + + case *FailCode: + if err := ReadElement(r, (*uint16)(e)); err != nil { + return err + } + + case *ChannelID: + if _, err := io.ReadFull(r, e[:]); err != nil { + return err + } + + case *ShortChannelID: + var blockHeight [4]byte + if _, err = io.ReadFull(r, blockHeight[1:]); err != nil { + return err + } + + var txIndex [4]byte + if _, err = io.ReadFull(r, txIndex[1:]); err != nil { + return err + } + + var txPosition [2]byte + if _, err = io.ReadFull(r, txPosition[:]); err != nil { + return err + } + + *e = ShortChannelID{ + BlockHeight: binary.BigEndian.Uint32(blockHeight[:]), + TxIndex: binary.BigEndian.Uint32(txIndex[:]), + TxPosition: binary.BigEndian.Uint16(txPosition[:]), + } + + case *ExtraOpaqueData: + return e.Decode(r) + + default: + return fmt.Errorf("unknown type in ReadElement: %T", e) + } + + return nil +} + +// ReadElements deserializes a variable number of elements into the passed +// io.Reader, with each element being deserialized according to the ReadElement +// function. +func ReadElements(r io.Reader, elements ...interface{}) error { + for _, element := range elements { + err := ReadElement(r, element) + if err != nil { + return err + } + } + return nil +} diff --git a/payments/db/migration1/lnwire/message.go b/payments/db/migration1/lnwire/message.go new file mode 100644 index 000000000..def762f18 --- /dev/null +++ b/payments/db/migration1/lnwire/message.go @@ -0,0 +1,323 @@ +// Copyright (c) 2013-2017 The btcsuite developers +// Copyright (c) 2015-2016 The Decred developers +// code derived from https://github .com/btcsuite/btcd/blob/master/wire/message.go +// Copyright (C) 2015-2022 The Lightning Network Developers + +package lnwire + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" +) + +// MessageTypeSize is the size in bytes of the message type field in the header +// of all messages. +const MessageTypeSize = 2 + +// MessageType is the unique 2 byte big-endian integer that indicates the type +// of message on the wire. All messages have a very simple header which +// consists simply of 2-byte message type. We omit a length field, and checksum +// as the Lightning Protocol is intended to be encapsulated within a +// confidential+authenticated cryptographic messaging protocol. +type MessageType uint16 + +// The currently defined message types within this current version of the +// Lightning protocol. +const ( + MsgWarning MessageType = 1 + MsgStfu = 2 + MsgInit = 16 + MsgError = 17 + MsgPing = 18 + MsgPong = 19 + MsgOpenChannel = 32 + MsgAcceptChannel = 33 + MsgFundingCreated = 34 + MsgFundingSigned = 35 + MsgChannelReady = 36 + MsgShutdown = 38 + MsgClosingSigned = 39 + MsgClosingComplete = 40 + MsgClosingSig = 41 + MsgDynPropose = 111 + MsgDynAck = 113 + MsgDynReject = 115 + MsgDynCommit = 117 + MsgUpdateAddHTLC = 128 + MsgUpdateFulfillHTLC = 130 + MsgUpdateFailHTLC = 131 + MsgCommitSig = 132 + MsgRevokeAndAck = 133 + MsgUpdateFee = 134 + MsgUpdateFailMalformedHTLC = 135 + MsgChannelReestablish = 136 + MsgChannelAnnouncement = 256 + MsgNodeAnnouncement = 257 + MsgChannelUpdate = 258 + MsgAnnounceSignatures = 259 + MsgAnnounceSignatures2 = 260 + MsgQueryShortChanIDs = 261 + MsgReplyShortChanIDsEnd = 262 + MsgQueryChannelRange = 263 + MsgReplyChannelRange = 264 + MsgGossipTimestampRange = 265 + MsgChannelAnnouncement2 = 267 + MsgNodeAnnouncement2 = 269 + MsgChannelUpdate2 = 271 + MsgOnionMessage = 513 + MsgKickoffSig = 777 + + // MsgEnd defines the end of the official message range of the protocol. + // If a new message is added beyond this message, then this should be + // modified. + MsgEnd = 778 +) + +// IsChannelUpdate is a filter function that discerns channel update messages +// from the other messages in the Lightning Network Protocol. +func (t MessageType) IsChannelUpdate() bool { + switch t { + case MsgUpdateAddHTLC: + return true + case MsgUpdateFulfillHTLC: + return true + case MsgUpdateFailHTLC: + return true + case MsgUpdateFailMalformedHTLC: + return true + case MsgUpdateFee: + return true + default: + return false + } +} + +// ErrorEncodeMessage is used when failed to encode the message payload. +func ErrorEncodeMessage(err error) error { + return fmt.Errorf("failed to encode message to buffer, got %w", err) +} + +// ErrorWriteMessageType is used when failed to write the message type. +func ErrorWriteMessageType(err error) error { + return fmt.Errorf("failed to write message type, got %w", err) +} + +// ErrorPayloadTooLarge is used when the payload size exceeds the +// MaxMsgBody. +func ErrorPayloadTooLarge(size int) error { + return fmt.Errorf( + "message payload is too large - encoded %d bytes, "+ + "but maximum message payload is %d bytes", + size, MaxMsgBody, + ) +} + +// String return the string representation of message type. +func (t MessageType) String() string { + switch t { + case MsgWarning: + return "Warning" + case MsgStfu: + return "Stfu" + case MsgInit: + return "Init" + case MsgOpenChannel: + return "MsgOpenChannel" + case MsgAcceptChannel: + return "MsgAcceptChannel" + case MsgFundingCreated: + return "MsgFundingCreated" + case MsgFundingSigned: + return "MsgFundingSigned" + case MsgChannelReady: + return "ChannelReady" + case MsgShutdown: + return "Shutdown" + case MsgClosingSigned: + return "ClosingSigned" + case MsgDynPropose: + return "DynPropose" + case MsgDynAck: + return "DynAck" + case MsgDynReject: + return "DynReject" + case MsgDynCommit: + return "DynCommit" + case MsgKickoffSig: + return "KickoffSig" + case MsgUpdateAddHTLC: + return "UpdateAddHTLC" + case MsgUpdateFailHTLC: + return "UpdateFailHTLC" + case MsgUpdateFulfillHTLC: + return "UpdateFulfillHTLC" + case MsgCommitSig: + return "CommitSig" + case MsgRevokeAndAck: + return "RevokeAndAck" + case MsgUpdateFailMalformedHTLC: + return "UpdateFailMalformedHTLC" + case MsgChannelReestablish: + return "ChannelReestablish" + case MsgError: + return "Error" + case MsgChannelAnnouncement: + return "ChannelAnnouncement" + case MsgChannelUpdate: + return "ChannelUpdate" + case MsgNodeAnnouncement: + return "NodeAnnouncement1" + case MsgPing: + return "Ping" + case MsgAnnounceSignatures: + return "AnnounceSignatures" + case MsgPong: + return "Pong" + case MsgUpdateFee: + return "UpdateFee" + case MsgQueryShortChanIDs: + return "QueryShortChanIDs" + case MsgReplyShortChanIDsEnd: + return "ReplyShortChanIDsEnd" + case MsgQueryChannelRange: + return "QueryChannelRange" + case MsgReplyChannelRange: + return "ReplyChannelRange" + case MsgGossipTimestampRange: + return "GossipTimestampRange" + case MsgClosingComplete: + return "ClosingComplete" + case MsgClosingSig: + return "ClosingSig" + case MsgAnnounceSignatures2: + return "MsgAnnounceSignatures2" + case MsgChannelAnnouncement2: + return "ChannelAnnouncement2" + case MsgNodeAnnouncement2: + return "NodeAnnouncement2" + case MsgChannelUpdate2: + return "ChannelUpdate2" + case MsgOnionMessage: + return "OnionMessage" + default: + return "" + } +} + +// UnknownMessage is an implementation of the error interface that allows the +// creation of an error in response to an unknown message. +type UnknownMessage struct { + messageType MessageType +} + +// Error returns a human readable string describing the error. +// +// This is part of the error interface. +func (u *UnknownMessage) Error() string { + return fmt.Sprintf("unable to parse message of unknown type: %v", + u.messageType) +} + +// Serializable is an interface which defines a lightning wire serializable +// object. +type Serializable interface { + // Decode reads the bytes stream and converts it to the object. + Decode(io.Reader, uint32) error + + // Encode converts object to the bytes stream and write it into the + // write buffer. + Encode(*bytes.Buffer, uint32) error +} + +// Message is an interface that defines a lightning wire protocol message. The +// interface is general in order to allow implementing types full control over +// the representation of its data. +type Message interface { + Serializable + MsgType() MessageType +} + +// LinkUpdater is an interface implemented by most messages in BOLT 2 that are +// allowed to update the channel state. +type LinkUpdater interface { + // All LinkUpdater messages are messages and so we embed the interface + // so that we can treat it as a message if all we know about it is that + // it is a LinkUpdater message. + Message + + // TargetChanID returns the channel id of the link for which this + // message is intended. + TargetChanID() ChannelID +} + +// SizeableMessage is an interface that extends the base Message interface with +// a method to calculate the serialized size of a message. +type SizeableMessage interface { + Message + + // SerializedSize returns the serialized size of the message in bytes. + // The returned size includes the message type header bytes. + SerializedSize() (uint32, error) +} + +// MessageSerializedSize calculates the serialized size of a message in bytes. +// This is a helper function that can be used by all message types to implement +// the SerializedSize method. +func MessageSerializedSize(msg Message) (uint32, error) { + var buf bytes.Buffer + + // Encode the message to the buffer. + if err := msg.Encode(&buf, 0); err != nil { + return 0, err + } + + // Add the size of the message type. + return uint32(buf.Len()) + MessageTypeSize, nil +} + +// WriteMessage writes a lightning Message to a buffer including the necessary +// header information and returns the number of bytes written. If any error is +// encountered, the buffer passed will be reset to its original state since we +// don't want any broken bytes left. In other words, no bytes will be written +// if there's an error. Either all or none of the message bytes will be written +// to the buffer. +// +// NOTE: this method is not concurrent safe. +func WriteMessage(buf *bytes.Buffer, msg Message, pver uint32) (int, error) { + // Record the size of the bytes already written in buffer. + oldByteSize := buf.Len() + + // cleanBrokenBytes is a helper closure that helps reset the buffer to + // its original state. It truncates all the bytes written in current + // scope. + var cleanBrokenBytes = func(b *bytes.Buffer) int { + b.Truncate(oldByteSize) + return 0 + } + + // Write the message type. + var mType [2]byte + binary.BigEndian.PutUint16(mType[:], uint16(msg.MsgType())) + msgTypeBytes, err := buf.Write(mType[:]) + if err != nil { + return cleanBrokenBytes(buf), ErrorWriteMessageType(err) + } + + // Use the write buffer to encode our message. + if err := msg.Encode(buf, pver); err != nil { + return cleanBrokenBytes(buf), ErrorEncodeMessage(err) + } + + // Enforce maximum overall message payload. The write buffer now has + // the size of len(originalBytes) + len(payload) + len(type). We want + // to enforce the payload here, so we subtract it by the length of the + // type and old bytes. + lenp := buf.Len() - oldByteSize - msgTypeBytes + if lenp > MaxMsgBody { + return cleanBrokenBytes(buf), ErrorPayloadTooLarge(lenp) + } + + return buf.Len() - oldByteSize, nil +} diff --git a/payments/db/migration1/lnwire/msat.go b/payments/db/migration1/lnwire/msat.go new file mode 100644 index 000000000..7d6d581e1 --- /dev/null +++ b/payments/db/migration1/lnwire/msat.go @@ -0,0 +1,92 @@ +package lnwire + +import ( + "fmt" + "io" + + "github.com/btcsuite/btcd/btcutil" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // mSatScale is a value that's used to scale satoshis to milli-satoshis, and + // the other way around. + mSatScale uint64 = 1000 + + // MaxMilliSatoshi is the maximum number of msats that can be expressed + // in this data type. + MaxMilliSatoshi = ^MilliSatoshi(0) +) + +// MilliSatoshi are the native unit of the Lightning Network. A milli-satoshi +// is simply 1/1000th of a satoshi. There are 1000 milli-satoshis in a single +// satoshi. Within the network, all HTLC payments are denominated in +// milli-satoshis. As milli-satoshis aren't deliverable on the native +// blockchain, before settling to broadcasting, the values are rounded down to +// the nearest satoshi. +type MilliSatoshi uint64 + +// NewMSatFromSatoshis creates a new MilliSatoshi instance from a target amount +// of satoshis. +func NewMSatFromSatoshis(sat btcutil.Amount) MilliSatoshi { + return MilliSatoshi(uint64(sat) * mSatScale) +} + +// ToBTC converts the target MilliSatoshi amount to its corresponding value +// when expressed in BTC. +func (m MilliSatoshi) ToBTC() float64 { + sat := m.ToSatoshis() + return sat.ToBTC() +} + +// ToSatoshis converts the target MilliSatoshi amount to satoshis. Simply, this +// sheds a factor of 1000 from the mSAT amount in order to convert it to SAT. +func (m MilliSatoshi) ToSatoshis() btcutil.Amount { + return btcutil.Amount(uint64(m) / mSatScale) +} + +// String returns the string representation of the mSAT amount. +func (m MilliSatoshi) String() string { + return fmt.Sprintf("%v mSAT", uint64(m)) +} + +// TODO(roasbeef): extend with arithmetic operations? + +// Record returns a TLV record that can be used to encode/decode a MilliSatoshi +// to/from a TLV stream. +func (m *MilliSatoshi) Record() tlv.Record { + msat := uint64(*m) + + return tlv.MakeDynamicRecord( + 0, m, tlv.SizeBigSize(&msat), encodeMilliSatoshis, + decodeMilliSatoshis, + ) +} + +func encodeMilliSatoshis(w io.Writer, val interface{}, buf *[8]byte) error { + if v, ok := val.(*MilliSatoshi); ok { + bigSize := uint64(*v) + + return tlv.EBigSize(w, &bigSize, buf) + } + + return tlv.NewTypeForEncodingErr(val, "lnwire.MilliSatoshi") +} + +func decodeMilliSatoshis(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if v, ok := val.(*MilliSatoshi); ok { + var bigSize uint64 + err := tlv.DBigSize(r, &bigSize, buf, l) + if err != nil { + return err + } + + *v = MilliSatoshi(bigSize) + + return nil + } + + return tlv.NewTypeForDecodingErr(val, "lnwire.MilliSatoshi", l, l) +} diff --git a/payments/db/migration1/lnwire/onion_error.go b/payments/db/migration1/lnwire/onion_error.go new file mode 100644 index 000000000..9cc115ff0 --- /dev/null +++ b/payments/db/migration1/lnwire/onion_error.go @@ -0,0 +1,1546 @@ +package lnwire + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/binary" + "fmt" + "io" + + "github.com/davecgh/go-spew/spew" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/tlv" +) + +// FailureMessage represents the onion failure object identified by its unique +// failure code. +type FailureMessage interface { + // Code returns a failure code describing the exact nature of the + // error. + Code() FailCode + + // Error returns a human readable string describing the error. With + // this method, the FailureMessage interface meets the built-in error + // interface. + Error() string +} + +// FailureMessageLength is the size of the failure message plus the size of +// padding. The FailureMessage message should always be EXACTLY this size. +const FailureMessageLength = 256 + +const ( + // FlagBadOnion error flag describes an unparsable, encrypted by + // previous node. + FlagBadOnion FailCode = 0x8000 + + // FlagPerm error flag indicates a permanent failure. + FlagPerm FailCode = 0x4000 + + // FlagNode error flag indicates a node failure. + FlagNode FailCode = 0x2000 + + // FlagUpdate error flag indicates a new channel update is enclosed + // within the error. + FlagUpdate FailCode = 0x1000 +) + +// FailCode specifies the precise reason that an upstream HTLC was canceled. +// Each UpdateFailHTLC message carries a FailCode which is to be passed +// backwards, encrypted at each step back to the source of the HTLC within the +// route. +type FailCode uint16 + +// The currently defined onion failure types within this current version of the +// Lightning protocol. +const ( + CodeNone FailCode = 0 + CodeInvalidRealm = FlagBadOnion | 1 + CodeTemporaryNodeFailure = FlagNode | 2 + CodePermanentNodeFailure = FlagPerm | FlagNode | 2 + CodeRequiredNodeFeatureMissing = FlagPerm | FlagNode | 3 + CodeInvalidOnionVersion = FlagBadOnion | FlagPerm | 4 + CodeInvalidOnionHmac = FlagBadOnion | FlagPerm | 5 + CodeInvalidOnionKey = FlagBadOnion | FlagPerm | 6 + CodeTemporaryChannelFailure = FlagUpdate | 7 + CodePermanentChannelFailure = FlagPerm | 8 + CodeRequiredChannelFeatureMissing = FlagPerm | 9 + CodeUnknownNextPeer = FlagPerm | 10 + CodeAmountBelowMinimum = FlagUpdate | 11 + CodeFeeInsufficient = FlagUpdate | 12 + CodeIncorrectCltvExpiry = FlagUpdate | 13 + CodeExpiryTooSoon = FlagUpdate | 14 + CodeChannelDisabled = FlagUpdate | 20 + CodeIncorrectOrUnknownPaymentDetails = FlagPerm | 15 + CodeIncorrectPaymentAmount = FlagPerm | 16 + CodeFinalExpiryTooSoon FailCode = 17 + CodeFinalIncorrectCltvExpiry FailCode = 18 + CodeFinalIncorrectHtlcAmount FailCode = 19 + CodeExpiryTooFar FailCode = 21 + CodeInvalidOnionPayload = FlagPerm | 22 + CodeMPPTimeout FailCode = 23 + CodeInvalidBlinding = FlagBadOnion | FlagPerm | 24 //nolint:ll +) + +// String returns the string representation of the failure code. +func (c FailCode) String() string { + switch c { + case CodeInvalidRealm: + return "InvalidRealm" + + case CodeTemporaryNodeFailure: + return "TemporaryNodeFailure" + + case CodePermanentNodeFailure: + return "PermanentNodeFailure" + + case CodeRequiredNodeFeatureMissing: + return "RequiredNodeFeatureMissing" + + case CodeInvalidOnionVersion: + return "InvalidOnionVersion" + + case CodeInvalidOnionHmac: + return "InvalidOnionHmac" + + case CodeInvalidOnionKey: + return "InvalidOnionKey" + + case CodeTemporaryChannelFailure: + return "TemporaryChannelFailure" + + case CodePermanentChannelFailure: + return "PermanentChannelFailure" + + case CodeRequiredChannelFeatureMissing: + return "RequiredChannelFeatureMissing" + + case CodeUnknownNextPeer: + return "UnknownNextPeer" + + case CodeAmountBelowMinimum: + return "AmountBelowMinimum" + + case CodeFeeInsufficient: + return "FeeInsufficient" + + case CodeIncorrectCltvExpiry: + return "IncorrectCltvExpiry" + + case CodeIncorrectPaymentAmount: + return "IncorrectPaymentAmount" + + case CodeExpiryTooSoon: + return "ExpiryTooSoon" + + case CodeChannelDisabled: + return "ChannelDisabled" + + case CodeIncorrectOrUnknownPaymentDetails: + return "IncorrectOrUnknownPaymentDetails" + + case CodeFinalExpiryTooSoon: + return "FinalExpiryTooSoon" + + case CodeFinalIncorrectCltvExpiry: + return "FinalIncorrectCltvExpiry" + + case CodeFinalIncorrectHtlcAmount: + return "FinalIncorrectHtlcAmount" + + case CodeExpiryTooFar: + return "ExpiryTooFar" + + case CodeInvalidOnionPayload: + return "InvalidOnionPayload" + + case CodeMPPTimeout: + return "MPPTimeout" + + case CodeInvalidBlinding: + return "InvalidBlinding" + + default: + return "" + } +} + +// FailInvalidRealm is returned if the realm byte is unknown. +// +// NOTE: May be returned by any node in the payment route. +type FailInvalidRealm struct{} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailInvalidRealm) Error() string { + return f.Code().String() +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailInvalidRealm) Code() FailCode { + return CodeInvalidRealm +} + +// FailTemporaryNodeFailure is returned if an otherwise unspecified transient +// error occurs for the entire node. +// +// NOTE: May be returned by any node in the payment route. +type FailTemporaryNodeFailure struct{} + +// Code returns the failure unique code. +// NOTE: Part of the FailureMessage interface. +func (f *FailTemporaryNodeFailure) Code() FailCode { + return CodeTemporaryNodeFailure +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailTemporaryNodeFailure) Error() string { + return f.Code().String() +} + +// FailPermanentNodeFailure is returned if an otherwise unspecified permanent +// error occurs for the entire node. +// +// NOTE: May be returned by any node in the payment route. +type FailPermanentNodeFailure struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailPermanentNodeFailure) Code() FailCode { + return CodePermanentNodeFailure +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailPermanentNodeFailure) Error() string { + return f.Code().String() +} + +// FailRequiredNodeFeatureMissing is returned if a node has requirement +// advertised in its node_announcement features which were not present in the +// onion. +// +// NOTE: May be returned by any node in the payment route. +type FailRequiredNodeFeatureMissing struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailRequiredNodeFeatureMissing) Code() FailCode { + return CodeRequiredNodeFeatureMissing +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailRequiredNodeFeatureMissing) Error() string { + return f.Code().String() +} + +// FailPermanentChannelFailure is return if an otherwise unspecified permanent +// error occurs for the outgoing channel (eg. channel (recently). +// +// NOTE: May be returned by any node in the payment route. +type FailPermanentChannelFailure struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailPermanentChannelFailure) Code() FailCode { + return CodePermanentChannelFailure +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailPermanentChannelFailure) Error() string { + return f.Code().String() +} + +// FailRequiredChannelFeatureMissing is returned if the outgoing channel has a +// requirement advertised in its channel announcement features which were not +// present in the onion. +// +// NOTE: May only be returned by intermediate nodes. +type FailRequiredChannelFeatureMissing struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailRequiredChannelFeatureMissing) Code() FailCode { + return CodeRequiredChannelFeatureMissing +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailRequiredChannelFeatureMissing) Error() string { + return f.Code().String() +} + +// FailUnknownNextPeer is returned if the next peer specified by the onion is +// not known. +// +// NOTE: May only be returned by intermediate nodes. +type FailUnknownNextPeer struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailUnknownNextPeer) Code() FailCode { + return CodeUnknownNextPeer +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailUnknownNextPeer) Error() string { + return f.Code().String() +} + +// FailIncorrectPaymentAmount is returned if the amount paid is less than the +// amount expected, the final node MUST fail the HTLC. If the amount paid is +// more than twice the amount expected, the final node SHOULD fail the HTLC. +// This allows the sender to reduce information leakage by altering the amount, +// without allowing accidental gross overpayment. +// +// NOTE: May only be returned by the final node in the path. +type FailIncorrectPaymentAmount struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailIncorrectPaymentAmount) Code() FailCode { + return CodeIncorrectPaymentAmount +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailIncorrectPaymentAmount) Error() string { + return f.Code().String() +} + +// FailIncorrectDetails is returned for two reasons: +// +// 1) if the payment hash has already been paid, the final node MAY treat the +// payment hash as unknown, or may succeed in accepting the HTLC. If the +// payment hash is unknown, the final node MUST fail the HTLC. +// +// 2) if the amount paid is less than the amount expected, the final node MUST +// fail the HTLC. If the amount paid is more than twice the amount expected, +// the final node SHOULD fail the HTLC. This allows the sender to reduce +// information leakage by altering the amount, without allowing accidental +// gross overpayment. +// +// NOTE: May only be returned by the final node in the path. +type FailIncorrectDetails struct { + // amount is the value of the extended HTLC. + amount MilliSatoshi + + // height is the block height when the htlc was received. + height uint32 + + // extraOpaqueData contains additional failure message tlv data. + extraOpaqueData ExtraOpaqueData +} + +// NewFailIncorrectDetails makes a new instance of the FailIncorrectDetails +// error bound to the specified HTLC amount and acceptance height. +func NewFailIncorrectDetails(amt MilliSatoshi, + height uint32) *FailIncorrectDetails { + + return &FailIncorrectDetails{ + amount: amt, + height: height, + extraOpaqueData: []byte{}, + } +} + +// Amount is the value of the extended HTLC. +func (f *FailIncorrectDetails) Amount() MilliSatoshi { + return f.amount +} + +// Height is the block height when the htlc was received. +func (f *FailIncorrectDetails) Height() uint32 { + return f.height +} + +// ExtraOpaqueData returns additional failure message tlv data. +func (f *FailIncorrectDetails) ExtraOpaqueData() ExtraOpaqueData { + return f.extraOpaqueData +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailIncorrectDetails) Code() FailCode { + return CodeIncorrectOrUnknownPaymentDetails +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailIncorrectDetails) Error() string { + return fmt.Sprintf( + "%v(amt=%v, height=%v)", CodeIncorrectOrUnknownPaymentDetails, + f.amount, f.height, + ) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailIncorrectDetails) Decode(r io.Reader, pver uint32) error { + err := ReadElement(r, &f.amount) + switch { + // This is an optional tack on that was added later in the protocol. As + // a result, older nodes may not include this value. We'll account for + // this by checking for io.EOF here which means that no bytes were read + // at all. + case err == io.EOF: + return nil + + case err != nil: + return err + } + + // At a later stage, the height field was also tacked on. We need to + // check for io.EOF here as well. + err = ReadElement(r, &f.height) + switch { + case err == io.EOF: + return nil + + case err != nil: + return err + } + + return f.extraOpaqueData.Decode(r) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailIncorrectDetails) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteMilliSatoshi(w, f.amount); err != nil { + return err + } + + if err := WriteUint32(w, f.height); err != nil { + return err + } + + return f.extraOpaqueData.Encode(w) +} + +// FailFinalExpiryTooSoon is returned if the cltv_expiry is too low, the final +// node MUST fail the HTLC. +// +// NOTE: May only be returned by the final node in the path. +type FailFinalExpiryTooSoon struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailFinalExpiryTooSoon) Code() FailCode { + return CodeFinalExpiryTooSoon +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailFinalExpiryTooSoon) Error() string { + return f.Code().String() +} + +// NewFinalExpiryTooSoon creates new instance of the FailFinalExpiryTooSoon. +func NewFinalExpiryTooSoon() *FailFinalExpiryTooSoon { + return &FailFinalExpiryTooSoon{} +} + +// FailInvalidOnionVersion is returned if the onion version byte is unknown. +// +// NOTE: May be returned only by intermediate nodes. +type FailInvalidOnionVersion struct { + // OnionSHA256 hash of the onion blob which haven't been proceeded. + OnionSHA256 [sha256.Size]byte +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailInvalidOnionVersion) Error() string { + return fmt.Sprintf("InvalidOnionVersion(onion_sha=%x)", f.OnionSHA256[:]) +} + +// NewInvalidOnionVersion creates new instance of the FailInvalidOnionVersion. +func NewInvalidOnionVersion(onion []byte) *FailInvalidOnionVersion { + return &FailInvalidOnionVersion{OnionSHA256: sha256.Sum256(onion)} +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailInvalidOnionVersion) Code() FailCode { + return CodeInvalidOnionVersion +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionVersion) Decode(r io.Reader, pver uint32) error { + return ReadElement(r, f.OnionSHA256[:]) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionVersion) Encode(w *bytes.Buffer, pver uint32) error { + return WriteBytes(w, f.OnionSHA256[:]) +} + +// FailInvalidOnionHmac is return if the onion HMAC is incorrect. +// +// NOTE: May only be returned by intermediate nodes. +type FailInvalidOnionHmac struct { + // OnionSHA256 hash of the onion blob which haven't been proceeded. + OnionSHA256 [sha256.Size]byte +} + +// NewInvalidOnionHmac creates new instance of the FailInvalidOnionHmac. +func NewInvalidOnionHmac(onion []byte) *FailInvalidOnionHmac { + return &FailInvalidOnionHmac{OnionSHA256: sha256.Sum256(onion)} +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailInvalidOnionHmac) Code() FailCode { + return CodeInvalidOnionHmac +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionHmac) Decode(r io.Reader, pver uint32) error { + return ReadElement(r, f.OnionSHA256[:]) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionHmac) Encode(w *bytes.Buffer, pver uint32) error { + return WriteBytes(w, f.OnionSHA256[:]) +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailInvalidOnionHmac) Error() string { + return fmt.Sprintf("InvalidOnionHMAC(onion_sha=%x)", f.OnionSHA256[:]) +} + +// FailInvalidOnionKey is return if the ephemeral key in the onion is +// unparsable. +// +// NOTE: May only be returned by intermediate nodes. +type FailInvalidOnionKey struct { + // OnionSHA256 hash of the onion blob which haven't been proceeded. + OnionSHA256 [sha256.Size]byte +} + +// NewInvalidOnionKey creates new instance of the FailInvalidOnionKey. +func NewInvalidOnionKey(onion []byte) *FailInvalidOnionKey { + return &FailInvalidOnionKey{OnionSHA256: sha256.Sum256(onion)} +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailInvalidOnionKey) Code() FailCode { + return CodeInvalidOnionKey +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionKey) Decode(r io.Reader, pver uint32) error { + return ReadElement(r, f.OnionSHA256[:]) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidOnionKey) Encode(w *bytes.Buffer, pver uint32) error { + return WriteBytes(w, f.OnionSHA256[:]) +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailInvalidOnionKey) Error() string { + return fmt.Sprintf("InvalidOnionKey(onion_sha=%x)", f.OnionSHA256[:]) +} + +// parseChannelUpdateCompatibilityMode will attempt to parse a channel updated +// encoded into an onion error payload in two ways. First, we'll try the +// compatibility oriented version wherein we'll _skip_ the length prefixing on +// the channel update message. Older versions of c-lighting do this so we'll +// attempt to parse these messages in order to retain compatibility. If we're +// unable to pull out a fully valid version, then we'll fall back to the +// regular parsing mechanism which includes the length prefix an NO type byte. +func parseChannelUpdateCompatibilityMode(reader io.Reader, length uint16, + chanUpdate *ChannelUpdate1, pver uint32) error { + + // Instantiate a LimitReader because there may be additional data + // present after the channel update. Without limiting the stream, the + // additional data would be interpreted as channel update tlv data. + limitReader := io.LimitReader(reader, int64(length)) + + r := bufio.NewReader(limitReader) + + // We'll peek out two bytes from the buffer without advancing the + // buffer so we can decide how to parse the remainder of it. + maybeTypeBytes, err := r.Peek(2) + if err != nil { + return err + } + + // Some nodes well prefix an additional set of bytes in front of their + // channel updates. These bytes will _almost_ always be 258 or the type + // of the ChannelUpdate message. + typeInt := binary.BigEndian.Uint16(maybeTypeBytes) + if typeInt == MsgChannelUpdate { + // At this point it's likely the case that this is a channel + // update message with its type prefixed, so we'll snip off the + // first two bytes and parse it as normal. + var throwAwayTypeBytes [2]byte + _, err := r.Read(throwAwayTypeBytes[:]) + if err != nil { + return err + } + } + + // At this pint, we've either decided to keep the entire thing, or snip + // off the first two bytes. In either case, we can just read it as + // normal. + return chanUpdate.Decode(r, pver) +} + +// FailTemporaryChannelFailure is if an otherwise unspecified transient error +// occurs for the outgoing channel (eg. channel capacity reached, too many +// in-flight htlcs) +// +// NOTE: May only be returned by intermediate nodes. +type FailTemporaryChannelFailure struct { + // Update is used to update information about state of the channel + // which caused the failure. + // + // NOTE: This field is optional. + Update *ChannelUpdate1 +} + +// NewTemporaryChannelFailure creates new instance of the FailTemporaryChannelFailure. +func NewTemporaryChannelFailure( + update *ChannelUpdate1) *FailTemporaryChannelFailure { + + return &FailTemporaryChannelFailure{Update: update} +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailTemporaryChannelFailure) Code() FailCode { + return CodeTemporaryChannelFailure +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailTemporaryChannelFailure) Error() string { + if f.Update == nil { + return f.Code().String() + } + + return fmt.Sprintf("TemporaryChannelFailure(update=%v)", + spew.Sdump(f.Update)) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailTemporaryChannelFailure) Decode(r io.Reader, pver uint32) error { + var length uint16 + err := ReadElement(r, &length) + if err != nil { + return err + } + + if length != 0 { + f.Update = &ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, f.Update, pver, + ) + } + + return nil +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailTemporaryChannelFailure) Encode(w *bytes.Buffer, + pver uint32) error { + + if f.Update != nil { + return writeOnionErrorChanUpdate(w, f.Update, pver) + } + + // Write zero length to indicate no channel_update is present. + return WriteUint16(w, 0) +} + +// FailAmountBelowMinimum is returned if the HTLC does not reach the current +// minimum amount, we tell them the amount of the incoming HTLC and the current +// channel setting for the outgoing channel. +// +// NOTE: May only be returned by the intermediate nodes in the path. +type FailAmountBelowMinimum struct { + // HtlcMsat is the wrong amount of the incoming HTLC. + HtlcMsat MilliSatoshi + + // Update is used to update information about state of the channel + // which caused the failure. + Update ChannelUpdate1 +} + +// NewAmountBelowMinimum creates new instance of the FailAmountBelowMinimum. +func NewAmountBelowMinimum(htlcMsat MilliSatoshi, + update ChannelUpdate1) *FailAmountBelowMinimum { + + return &FailAmountBelowMinimum{ + HtlcMsat: htlcMsat, + Update: update, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailAmountBelowMinimum) Code() FailCode { + return CodeAmountBelowMinimum +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailAmountBelowMinimum) Error() string { + return fmt.Sprintf("AmountBelowMinimum(amt=%v, update=%v", f.HtlcMsat, + spew.Sdump(f.Update)) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailAmountBelowMinimum) Decode(r io.Reader, pver uint32) error { + if err := ReadElement(r, &f.HtlcMsat); err != nil { + return err + } + + var length uint16 + if err := ReadElement(r, &length); err != nil { + return err + } + + f.Update = ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, &f.Update, pver, + ) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailAmountBelowMinimum) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteMilliSatoshi(w, f.HtlcMsat); err != nil { + return err + } + + return writeOnionErrorChanUpdate(w, &f.Update, pver) +} + +// FailFeeInsufficient is returned if the HTLC does not pay sufficient fee, we +// tell them the amount of the incoming HTLC and the current channel setting +// for the outgoing channel. +// +// NOTE: May only be returned by intermediate nodes. +type FailFeeInsufficient struct { + // HtlcMsat is the wrong amount of the incoming HTLC. + HtlcMsat MilliSatoshi + + // Update is used to update information about state of the channel + // which caused the failure. + Update ChannelUpdate1 +} + +// NewFeeInsufficient creates new instance of the FailFeeInsufficient. +func NewFeeInsufficient(htlcMsat MilliSatoshi, + update ChannelUpdate1) *FailFeeInsufficient { + return &FailFeeInsufficient{ + HtlcMsat: htlcMsat, + Update: update, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailFeeInsufficient) Code() FailCode { + return CodeFeeInsufficient +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailFeeInsufficient) Error() string { + return fmt.Sprintf("FeeInsufficient(htlc_amt==%v, update=%v", f.HtlcMsat, + spew.Sdump(f.Update)) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFeeInsufficient) Decode(r io.Reader, pver uint32) error { + if err := ReadElement(r, &f.HtlcMsat); err != nil { + return err + } + + var length uint16 + if err := ReadElement(r, &length); err != nil { + return err + } + + f.Update = ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, &f.Update, pver, + ) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFeeInsufficient) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteMilliSatoshi(w, f.HtlcMsat); err != nil { + return err + } + + return writeOnionErrorChanUpdate(w, &f.Update, pver) +} + +// FailIncorrectCltvExpiry is returned if outgoing cltv value does not match +// the update add htlc's cltv expiry minus cltv expiry delta for the outgoing +// channel, we tell them the cltv expiry and the current channel setting for +// the outgoing channel. +// +// NOTE: May only be returned by intermediate nodes. +type FailIncorrectCltvExpiry struct { + // CltvExpiry is the wrong absolute timeout in blocks, after which + // outgoing HTLC expires. + CltvExpiry uint32 + + // Update is used to update information about state of the channel + // which caused the failure. + Update ChannelUpdate1 +} + +// NewIncorrectCltvExpiry creates new instance of the FailIncorrectCltvExpiry. +func NewIncorrectCltvExpiry(cltvExpiry uint32, + update ChannelUpdate1) *FailIncorrectCltvExpiry { + + return &FailIncorrectCltvExpiry{ + CltvExpiry: cltvExpiry, + Update: update, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailIncorrectCltvExpiry) Code() FailCode { + return CodeIncorrectCltvExpiry +} + +func (f *FailIncorrectCltvExpiry) Error() string { + return fmt.Sprintf("IncorrectCltvExpiry(expiry=%v, update=%v", + f.CltvExpiry, spew.Sdump(f.Update)) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailIncorrectCltvExpiry) Decode(r io.Reader, pver uint32) error { + if err := ReadElement(r, &f.CltvExpiry); err != nil { + return err + } + + var length uint16 + if err := ReadElement(r, &length); err != nil { + return err + } + + f.Update = ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, &f.Update, pver, + ) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailIncorrectCltvExpiry) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteUint32(w, f.CltvExpiry); err != nil { + return err + } + + return writeOnionErrorChanUpdate(w, &f.Update, pver) +} + +// FailExpiryTooSoon is returned if the ctlv-expiry is too near, we tell them +// the current channel setting for the outgoing channel. +// +// NOTE: May only be returned by intermediate nodes. +type FailExpiryTooSoon struct { + // Update is used to update information about state of the channel + // which caused the failure. + Update ChannelUpdate1 +} + +// NewExpiryTooSoon creates new instance of the FailExpiryTooSoon. +func NewExpiryTooSoon(update ChannelUpdate1) *FailExpiryTooSoon { + return &FailExpiryTooSoon{ + Update: update, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailExpiryTooSoon) Code() FailCode { + return CodeExpiryTooSoon +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailExpiryTooSoon) Error() string { + return fmt.Sprintf("ExpiryTooSoon(update=%v", spew.Sdump(f.Update)) +} + +// Decode decodes the failure from l stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailExpiryTooSoon) Decode(r io.Reader, pver uint32) error { + var length uint16 + if err := ReadElement(r, &length); err != nil { + return err + } + + f.Update = ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, &f.Update, pver, + ) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailExpiryTooSoon) Encode(w *bytes.Buffer, pver uint32) error { + return writeOnionErrorChanUpdate(w, &f.Update, pver) +} + +// FailChannelDisabled is returned if the channel is disabled, we tell them the +// current channel setting for the outgoing channel. +// +// NOTE: May only be returned by intermediate nodes. +type FailChannelDisabled struct { + // Flags least-significant bit must be set to 0 if the creating node + // corresponds to the first node in the previously sent channel + // announcement and 1 otherwise. + Flags uint16 + + // Update is used to update information about state of the channel + // which caused the failure. + Update ChannelUpdate1 +} + +// NewChannelDisabled creates new instance of the FailChannelDisabled. +func NewChannelDisabled(flags uint16, + update ChannelUpdate1) *FailChannelDisabled { + + return &FailChannelDisabled{ + Flags: flags, + Update: update, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailChannelDisabled) Code() FailCode { + return CodeChannelDisabled +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailChannelDisabled) Error() string { + return fmt.Sprintf("ChannelDisabled(flags=%v, update=%v", f.Flags, + spew.Sdump(f.Update)) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailChannelDisabled) Decode(r io.Reader, pver uint32) error { + if err := ReadElement(r, &f.Flags); err != nil { + return err + } + + var length uint16 + if err := ReadElement(r, &length); err != nil { + return err + } + + f.Update = ChannelUpdate1{} + + return parseChannelUpdateCompatibilityMode( + r, length, &f.Update, pver, + ) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailChannelDisabled) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteUint16(w, f.Flags); err != nil { + return err + } + + return writeOnionErrorChanUpdate(w, &f.Update, pver) +} + +// FailFinalIncorrectCltvExpiry is returned if the outgoing_cltv_value does not +// match the ctlv_expiry of the HTLC at the final hop. +// +// NOTE: might be returned by final node only. +type FailFinalIncorrectCltvExpiry struct { + // CltvExpiry is the wrong absolute timeout in blocks, after which + // outgoing HTLC expires. + CltvExpiry uint32 +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailFinalIncorrectCltvExpiry) Error() string { + return fmt.Sprintf("FinalIncorrectCltvExpiry(expiry=%v)", f.CltvExpiry) +} + +// NewFinalIncorrectCltvExpiry creates new instance of the +// FailFinalIncorrectCltvExpiry. +func NewFinalIncorrectCltvExpiry(cltvExpiry uint32) *FailFinalIncorrectCltvExpiry { + return &FailFinalIncorrectCltvExpiry{ + CltvExpiry: cltvExpiry, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailFinalIncorrectCltvExpiry) Code() FailCode { + return CodeFinalIncorrectCltvExpiry +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFinalIncorrectCltvExpiry) Decode(r io.Reader, pver uint32) error { + return ReadElement(r, &f.CltvExpiry) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFinalIncorrectCltvExpiry) Encode(w *bytes.Buffer, + pver uint32) error { + + return WriteUint32(w, f.CltvExpiry) +} + +// FailFinalIncorrectHtlcAmount is returned if the amt_to_forward is higher +// than incoming_htlc_amt of the HTLC at the final hop. +// +// NOTE: May only be returned by the final node. +type FailFinalIncorrectHtlcAmount struct { + // IncomingHTLCAmount is the wrong forwarded htlc amount. + IncomingHTLCAmount MilliSatoshi +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailFinalIncorrectHtlcAmount) Error() string { + return fmt.Sprintf("FinalIncorrectHtlcAmount(amt=%v)", + f.IncomingHTLCAmount) +} + +// NewFinalIncorrectHtlcAmount creates new instance of the +// FailFinalIncorrectHtlcAmount. +func NewFinalIncorrectHtlcAmount(amount MilliSatoshi) *FailFinalIncorrectHtlcAmount { + return &FailFinalIncorrectHtlcAmount{ + IncomingHTLCAmount: amount, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailFinalIncorrectHtlcAmount) Code() FailCode { + return CodeFinalIncorrectHtlcAmount +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFinalIncorrectHtlcAmount) Decode(r io.Reader, pver uint32) error { + return ReadElement(r, &f.IncomingHTLCAmount) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailFinalIncorrectHtlcAmount) Encode(w *bytes.Buffer, + pver uint32) error { + + return WriteMilliSatoshi(w, f.IncomingHTLCAmount) +} + +// FailExpiryTooFar is returned if the CLTV expiry in the HTLC is too far in the +// future. +// +// NOTE: May be returned by any node in the payment route. +type FailExpiryTooFar struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailExpiryTooFar) Code() FailCode { + return CodeExpiryTooFar +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailExpiryTooFar) Error() string { + return f.Code().String() +} + +// InvalidOnionPayload is returned if the hop could not process the TLV payload +// enclosed in the onion. +type InvalidOnionPayload struct { + // Type is the TLV type that caused the specific failure. + Type uint64 + + // Offset is the byte offset within the payload where the failure + // occurred. + Offset uint16 +} + +// NewInvalidOnionPayload initializes a new InvalidOnionPayload failure. +func NewInvalidOnionPayload(typ uint64, offset uint16) *InvalidOnionPayload { + return &InvalidOnionPayload{ + Type: typ, + Offset: offset, + } +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *InvalidOnionPayload) Code() FailCode { + return CodeInvalidOnionPayload +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *InvalidOnionPayload) Error() string { + return fmt.Sprintf("%v(type=%v, offset=%d)", + f.Code(), f.Type, f.Offset) +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *InvalidOnionPayload) Decode(r io.Reader, pver uint32) error { + var buf [8]byte + typ, err := tlv.ReadVarInt(r, &buf) + if err != nil { + return err + } + f.Type = typ + + return ReadElements(r, &f.Offset) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *InvalidOnionPayload) Encode(w *bytes.Buffer, pver uint32) error { + var buf [8]byte + if err := tlv.WriteVarInt(w, f.Type, &buf); err != nil { + return err + } + + return WriteUint16(w, f.Offset) +} + +// FailMPPTimeout is returned if the complete amount for a multi part payment +// was not received within a reasonable time. +// +// NOTE: May only be returned by the final node in the path. +type FailMPPTimeout struct{} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailMPPTimeout) Code() FailCode { + return CodeMPPTimeout +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailMPPTimeout) Error() string { + return f.Code().String() +} + +// FailInvalidBlinding is returned if there has been a route blinding related +// error. +type FailInvalidBlinding struct { + OnionSHA256 [sha256.Size]byte +} + +// Code returns the failure unique code. +// +// NOTE: Part of the FailureMessage interface. +func (f *FailInvalidBlinding) Code() FailCode { + return CodeInvalidBlinding +} + +// Returns a human readable string describing the target FailureMessage. +// +// NOTE: Implements the error interface. +func (f *FailInvalidBlinding) Error() string { + return f.Code().String() +} + +// Decode decodes the failure from bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidBlinding) Decode(r io.Reader, _ uint32) error { + return ReadElement(r, f.OnionSHA256[:]) +} + +// Encode writes the failure in bytes stream. +// +// NOTE: Part of the Serializable interface. +func (f *FailInvalidBlinding) Encode(w *bytes.Buffer, _ uint32) error { + return WriteBytes(w, f.OnionSHA256[:]) +} + +// NewInvalidBlinding creates new instance of FailInvalidBlinding. +func NewInvalidBlinding( + onion fn.Option[[OnionPacketSize]byte]) *FailInvalidBlinding { + // The spec allows empty onion hashes for invalid blinding, so we only + // include our onion hash if it's provided. + if onion.IsNone() { + return &FailInvalidBlinding{} + } + + shaSum := fn.MapOptionZ(onion, func(o [OnionPacketSize]byte) [32]byte { + return sha256.Sum256(o[:]) + }) + + return &FailInvalidBlinding{OnionSHA256: shaSum} +} + +// DecodeFailure decodes, validates, and parses the lnwire onion failure, for +// the provided protocol version. +func DecodeFailure(r io.Reader, pver uint32) (FailureMessage, error) { + // First, we'll parse out the encapsulated failure message itself. This + // is a 2 byte length followed by the payload itself. + var failureLength uint16 + if err := ReadElement(r, &failureLength); err != nil { + return nil, fmt.Errorf("unable to read failure len: %w", err) + } + + failureData := make([]byte, failureLength) + if _, err := io.ReadFull(r, failureData); err != nil { + return nil, fmt.Errorf("unable to full read payload of "+ + "%v: %w", failureLength, err) + } + + // Read the padding. + var padLength uint16 + if err := ReadElement(r, &padLength); err != nil { + return nil, fmt.Errorf("unable to read pad len: %w", err) + } + + if _, err := io.CopyN(io.Discard, r, int64(padLength)); err != nil { + return nil, fmt.Errorf("unable to read padding %w", err) + } + + // Verify that we are at the end of the stream now. + scratch := make([]byte, 1) + _, err := r.Read(scratch) + if err != io.EOF { + return nil, fmt.Errorf("unexpected failure bytes") + } + + // Check the total length. Convert to 32 bits to prevent overflow. + totalLength := uint32(padLength) + uint32(failureLength) + if totalLength < FailureMessageLength { + return nil, fmt.Errorf("failure message too short: "+ + "msg=%v, pad=%v, total=%v", + failureLength, padLength, totalLength) + } + + // Decode the failure message. + dataReader := bytes.NewReader(failureData) + + return DecodeFailureMessage(dataReader, pver) +} + +// DecodeFailureMessage decodes just the failure message, ignoring any padding +// that may be present at the end. +func DecodeFailureMessage(r io.Reader, pver uint32) (FailureMessage, error) { + // Once we have the failure data, we can obtain the failure code from + // the first two bytes of the buffer. + var codeBytes [2]byte + if _, err := io.ReadFull(r, codeBytes[:]); err != nil { + return nil, fmt.Errorf("unable to read failure code: %w", err) + } + failCode := FailCode(binary.BigEndian.Uint16(codeBytes[:])) + + // Create the empty failure by given code and populate the failure with + // additional data if needed. + failure, err := makeEmptyOnionError(failCode) + if err != nil { + return nil, fmt.Errorf("unable to make empty error: %w", err) + } + + // Finally, if this failure has a payload, then we'll read that now as + // well. + switch f := failure.(type) { + case Serializable: + if err := f.Decode(r, pver); err != nil { + return nil, fmt.Errorf("unable to decode error "+ + "update (type=%T): %w", failure, err) + } + } + + return failure, nil +} + +// EncodeFailure encodes, including the necessary onion failure header +// information. +func EncodeFailure(w *bytes.Buffer, failure FailureMessage, pver uint32) error { + var failureMessageBuffer bytes.Buffer + + err := EncodeFailureMessage(&failureMessageBuffer, failure, pver) + if err != nil { + return err + } + + // The combined size of this message must be below the max allowed + // failure message length. + failureMessage := failureMessageBuffer.Bytes() + if len(failureMessage) > FailureMessageLength { + return fmt.Errorf("failure message exceed max "+ + "available size: %v", len(failureMessage)) + } + + // Finally, we'll add some padding in order to ensure that all failure + // messages are fixed size. + pad := make([]byte, FailureMessageLength-len(failureMessage)) + + if err := WriteUint16(w, uint16(len(failureMessage))); err != nil { + return err + } + + if err := WriteBytes(w, failureMessage); err != nil { + return err + } + if err := WriteUint16(w, uint16(len(pad))); err != nil { + return err + } + + return WriteBytes(w, pad) +} + +// EncodeFailureMessage encodes just the failure message without adding a length +// and padding the message for the onion protocol. +func EncodeFailureMessage(w *bytes.Buffer, + failure FailureMessage, pver uint32) error { + + // First, we'll write out the error code itself into the failure + // buffer. + var codeBytes [2]byte + code := uint16(failure.Code()) + binary.BigEndian.PutUint16(codeBytes[:], code) + _, err := w.Write(codeBytes[:]) + if err != nil { + return err + } + + // Next, some message have an additional message payload, if this is + // one of those types, then we'll also encode the error payload as + // well. + switch failure := failure.(type) { + case Serializable: + if err := failure.Encode(w, pver); err != nil { + return err + } + } + + return nil +} + +// makeEmptyOnionError creates a new empty onion error of the proper concrete +// type based on the passed failure code. +func makeEmptyOnionError(code FailCode) (FailureMessage, error) { + switch code { + case CodeInvalidRealm: + return &FailInvalidRealm{}, nil + + case CodeTemporaryNodeFailure: + return &FailTemporaryNodeFailure{}, nil + + case CodePermanentNodeFailure: + return &FailPermanentNodeFailure{}, nil + + case CodeRequiredNodeFeatureMissing: + return &FailRequiredNodeFeatureMissing{}, nil + + case CodePermanentChannelFailure: + return &FailPermanentChannelFailure{}, nil + + case CodeRequiredChannelFeatureMissing: + return &FailRequiredChannelFeatureMissing{}, nil + + case CodeUnknownNextPeer: + return &FailUnknownNextPeer{}, nil + + case CodeIncorrectOrUnknownPaymentDetails: + return &FailIncorrectDetails{}, nil + + case CodeIncorrectPaymentAmount: + return &FailIncorrectPaymentAmount{}, nil + + case CodeFinalExpiryTooSoon: + return &FailFinalExpiryTooSoon{}, nil + + case CodeInvalidOnionVersion: + return &FailInvalidOnionVersion{}, nil + + case CodeInvalidOnionHmac: + return &FailInvalidOnionHmac{}, nil + + case CodeInvalidOnionKey: + return &FailInvalidOnionKey{}, nil + + case CodeTemporaryChannelFailure: + return &FailTemporaryChannelFailure{}, nil + + case CodeAmountBelowMinimum: + return &FailAmountBelowMinimum{}, nil + + case CodeFeeInsufficient: + return &FailFeeInsufficient{}, nil + + case CodeIncorrectCltvExpiry: + return &FailIncorrectCltvExpiry{}, nil + + case CodeExpiryTooSoon: + return &FailExpiryTooSoon{}, nil + + case CodeChannelDisabled: + return &FailChannelDisabled{}, nil + + case CodeFinalIncorrectCltvExpiry: + return &FailFinalIncorrectCltvExpiry{}, nil + + case CodeFinalIncorrectHtlcAmount: + return &FailFinalIncorrectHtlcAmount{}, nil + + case CodeExpiryTooFar: + return &FailExpiryTooFar{}, nil + + case CodeInvalidOnionPayload: + return &InvalidOnionPayload{}, nil + + case CodeMPPTimeout: + return &FailMPPTimeout{}, nil + + case CodeInvalidBlinding: + return &FailInvalidBlinding{}, nil + + default: + return nil, fmt.Errorf("unknown error code: %v", code) + } +} + +// writeOnionErrorChanUpdate writes out a ChannelUpdate using the onion error +// format. The format is that we first write out the true serialized length of +// the channel update, followed by the serialized channel update itself. +func writeOnionErrorChanUpdate(w *bytes.Buffer, chanUpdate *ChannelUpdate1, + pver uint32) error { + + // First, we encode the channel update in a temporary buffer in order + // to get the exact serialized size. + var b bytes.Buffer + updateLen, err := WriteMessage(&b, chanUpdate, pver) + if err != nil { + return err + } + + // Now that we know the size, we can write the length out in the main + // writer. + if err := WriteUint16(w, uint16(updateLen)); err != nil { + return err + } + + // With the length written, we'll then write out the serialized channel + // update. + if _, err := w.Write(b.Bytes()); err != nil { + return err + } + + return nil +} diff --git a/payments/db/migration1/lnwire/short_channel_id.go b/payments/db/migration1/lnwire/short_channel_id.go new file mode 100644 index 000000000..e26575001 --- /dev/null +++ b/payments/db/migration1/lnwire/short_channel_id.go @@ -0,0 +1,105 @@ +package lnwire + +import ( + "fmt" + "io" + + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // AliasScidRecordType is the type of the experimental record to denote + // the alias being used in an option_scid_alias channel. + AliasScidRecordType tlv.Type = 1 +) + +// ShortChannelID represents the set of data which is needed to retrieve all +// necessary data to validate the channel existence. +type ShortChannelID struct { + // BlockHeight is the height of the block where funding transaction + // located. + // + // NOTE: This field is limited to 3 bytes. + BlockHeight uint32 + + // TxIndex is a position of funding transaction within a block. + // + // NOTE: This field is limited to 3 bytes. + TxIndex uint32 + + // TxPosition indicating transaction output which pays to the channel. + TxPosition uint16 +} + +// NewShortChanIDFromInt returns a new ShortChannelID which is the decoded +// version of the compact channel ID encoded within the uint64. The format of +// the compact channel ID is as follows: 3 bytes for the block height, 3 bytes +// for the transaction index, and 2 bytes for the output index. +func NewShortChanIDFromInt(chanID uint64) ShortChannelID { + return ShortChannelID{ + BlockHeight: uint32(chanID >> 40), + TxIndex: uint32(chanID>>16) & 0xFFFFFF, + TxPosition: uint16(chanID), + } +} + +// ToUint64 converts the ShortChannelID into a compact format encoded within a +// uint64 (8 bytes). +func (c ShortChannelID) ToUint64() uint64 { + // TODO(roasbeef): explicit error on overflow? + return ((uint64(c.BlockHeight) << 40) | (uint64(c.TxIndex) << 16) | + (uint64(c.TxPosition))) +} + +// String generates a human-readable representation of the channel ID. +func (c ShortChannelID) String() string { + return fmt.Sprintf("%d:%d:%d", c.BlockHeight, c.TxIndex, c.TxPosition) +} + +// AltString generates a human-readable representation of the channel ID +// with 'x' as a separator. +func (c ShortChannelID) AltString() string { + return fmt.Sprintf("%dx%dx%d", c.BlockHeight, c.TxIndex, c.TxPosition) +} + +// Record returns a TLV record that can be used to encode/decode a +// ShortChannelID to/from a TLV stream. +func (c *ShortChannelID) Record() tlv.Record { + return tlv.MakeStaticRecord( + AliasScidRecordType, c, 8, EShortChannelID, DShortChannelID, + ) +} + +// IsDefault returns true if the ShortChannelID represents the zero value for +// its type. +func (c ShortChannelID) IsDefault() bool { + return c == ShortChannelID{} +} + +// EShortChannelID is an encoder for ShortChannelID. It is exported so other +// packages can use the encoding scheme. +func EShortChannelID(w io.Writer, val interface{}, buf *[8]byte) error { + if v, ok := val.(*ShortChannelID); ok { + return tlv.EUint64T(w, v.ToUint64(), buf) + } + return tlv.NewTypeForEncodingErr(val, "lnwire.ShortChannelID") +} + +// DShortChannelID is a decoder for ShortChannelID. It is exported so other +// packages can use the decoding scheme. +func DShortChannelID(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if v, ok := val.(*ShortChannelID); ok { + var scid uint64 + // tlv.DUint64 forces the length to be 8 bytes. + err := tlv.DUint64(r, &scid, buf, l) + if err != nil { + return err + } + + *v = NewShortChanIDFromInt(scid) + return nil + } + return tlv.NewTypeForDecodingErr(val, "lnwire.ShortChannelID", l, 8) +} diff --git a/payments/db/migration1/lnwire/signature.go b/payments/db/migration1/lnwire/signature.go new file mode 100644 index 000000000..35b039ed0 --- /dev/null +++ b/payments/db/migration1/lnwire/signature.go @@ -0,0 +1,292 @@ +package lnwire + +import ( + "errors" + "fmt" + + "github.com/btcsuite/btcd/btcec/v2/ecdsa" + "github.com/btcsuite/btcd/btcec/v2/schnorr" + "github.com/lightningnetwork/lnd/input" + "github.com/lightningnetwork/lnd/tlv" +) + +var ( + errSigTooShort = errors.New("malformed signature: too short") + errBadLength = errors.New("malformed signature: bad length") + errBadRLength = errors.New("malformed signature: bogus R length") + errBadSLength = errors.New("malformed signature: bogus S length") + errRTooLong = errors.New("R is over 32 bytes long without padding") + errSTooLong = errors.New("S is over 32 bytes long without padding") +) + +// sigType represents the type of signature that is carried within the Sig. +// Today this can either be an ECDSA sig or a schnorr sig. Both of these can +// fit cleanly into 64 bytes. +type sigType uint + +const ( + // sigTypeECDSA represents an ECDSA signature. + sigTypeECDSA sigType = iota + + // sigTypeSchnorr represents a schnorr signature. + sigTypeSchnorr +) + +// Sig is a fixed-sized ECDSA signature or 64-byte schnorr signature. For the +// ECDSA sig, unlike Bitcoin, we use fixed sized signatures on the wire, +// instead of DER encoded signatures. This type provides several methods to +// convert to/from a regular Bitcoin DER encoded signature (raw bytes and +// *ecdsa.Signature). +type Sig struct { + bytes [64]byte + + sigType sigType +} + +// ForceSchnorr forces the signature to be interpreted as a schnorr signature. +// This is useful when reading an HTLC sig off the wire for a taproot channel. +// In this case, in order to obtain an input.Signature, we need to know that +// the sig is a schnorr sig. +func (s *Sig) ForceSchnorr() { + s.sigType = sigTypeSchnorr +} + +// RawBytes returns the raw bytes of signature. +func (s *Sig) RawBytes() []byte { + return s.bytes[:] +} + +// Copy copies the signature into a new Sig instance. +func (s *Sig) Copy() Sig { + var sCopy Sig + copy(sCopy.bytes[:], s.bytes[:]) + sCopy.sigType = s.sigType + + return sCopy +} + +// Record returns a Record that can be used to encode or decode the backing +// object. +// +// This returns a record that serializes the sig as a 64-byte fixed size +// signature. +func (s *Sig) Record() tlv.Record { + // We set a type here as zero as it isn't needed when used as a + // RecordT. + return tlv.MakePrimitiveRecord(0, &s.bytes) +} + +// NewSigFromWireECDSA returns a Sig instance based on an ECDSA signature +// that's already in the 64-byte format we expect. +func NewSigFromWireECDSA(sig []byte) (Sig, error) { + if len(sig) != 64 { + return Sig{}, fmt.Errorf("%w: %v bytes", errSigTooShort, + len(sig)) + } + + var s Sig + copy(s.bytes[:], sig) + + return s, nil +} + +// NewSigFromECDSARawSignature returns a Sig from a Bitcoin raw signature +// encoded in the canonical DER encoding. +func NewSigFromECDSARawSignature(sig []byte) (Sig, error) { + var b [64]byte + + // Check the total length is above the minimal. + if len(sig) < ecdsa.MinSigLen { + return Sig{}, errSigTooShort + } + + // The DER representation is laid out as: + // 0x30 0x02 r 0x02 s + // which means the length of R is the 4th byte and the length of S is + // the second byte after R ends. 0x02 signifies a length-prefixed, + // zero-padded, big-endian bigint. 0x30 signifies a DER signature. + // See the Serialize() method for ecdsa.Signature for details. + + // Reading , remaining: [0x02 r 0x02 s] + sigLen := int(sig[1]) + + // siglen should be less than the entire message and greater than + // the minimal message size. + if sigLen+2 > len(sig) || sigLen+2 < ecdsa.MinSigLen { + return Sig{}, errBadLength + } + + // Reading , remaining: [r 0x02 s] + rLen := int(sig[3]) + + // rLen must be positive and must be able to fit in other elements. + // Assuming s is one byte, then we have 0x30, , 0x20, + // , 0x20, , s, a total of 7 bytes. + if rLen <= 0 || rLen+7 > len(sig) { + return Sig{}, errBadRLength + } + + // Reading , remaining: [s] + sLen := int(sig[5+rLen]) + + // S should be the rest of the string. + // sLen must be positive and must be able to fit in other elements. + // We know r is rLen bytes, and we have 0x30, , 0x20, + // , 0x20, , a total of rLen+6 bytes. + if sLen <= 0 || sLen+rLen+6 > len(sig) { + return Sig{}, errBadSLength + } + + // Check to make sure R and S can both fit into their intended buffers. + // We check S first because these code blocks decrement sLen and rLen + // in the case of a 33-byte 0-padded integer returned from Serialize() + // and rLen is used in calculating array indices for S. We can track + // this with additional variables, but it's more efficient to just + // check S first. + if sLen > 32 { + if (sLen > 33) || (sig[6+rLen] != 0x00) { + return Sig{}, errSTooLong + } + sLen-- + copy(b[64-sLen:], sig[7+rLen:]) + } else { + copy(b[64-sLen:], sig[6+rLen:]) + } + + // Do the same for R as we did for S + if rLen > 32 { + if (rLen > 33) || (sig[4] != 0x00) { + return Sig{}, errRTooLong + } + rLen-- + copy(b[32-rLen:], sig[5:5+rLen]) + } else { + copy(b[32-rLen:], sig[4:4+rLen]) + } + + return Sig{ + bytes: b, + sigType: sigTypeECDSA, + }, nil +} + +// NewSigFromSchnorrRawSignature converts a raw schnorr signature into an +// lnwire.Sig. +func NewSigFromSchnorrRawSignature(sig []byte) (Sig, error) { + var s Sig + copy(s.bytes[:], sig) + s.sigType = sigTypeSchnorr + + return s, nil +} + +// NewSigFromSignature creates a new signature as used on the wire, from an +// existing ecdsa.Signature or schnorr.Signature. +func NewSigFromSignature(e input.Signature) (Sig, error) { + if e == nil { + return Sig{}, fmt.Errorf("cannot decode empty signature") + } + + // Nil is still a valid interface, apparently. So we need a more + // explicit check here. + if ecsig, ok := e.(*ecdsa.Signature); ok && ecsig == nil { + return Sig{}, fmt.Errorf("cannot decode empty signature") + } + + switch ecSig := e.(type) { + // If this is a schnorr signature, then we can just pack it as normal, + // since the default encoding is already 64 bytes. + case *schnorr.Signature: + return NewSigFromSchnorrRawSignature(e.Serialize()) + + // For ECDSA signatures, we'll need to do a bit more work to map the + // signature into a compact 64 byte form. + case *ecdsa.Signature: + // Serialize the signature with all the checks that entails. + return NewSigFromECDSARawSignature(e.Serialize()) + + default: + return Sig{}, fmt.Errorf("unknown wire sig type: %T", ecSig) + } +} + +// ToSignature converts the fixed-sized signature to a input.Signature which +// can be used for signature validation checks. +func (s *Sig) ToSignature() (input.Signature, error) { + switch s.sigType { + case sigTypeSchnorr: + return schnorr.ParseSignature(s.bytes[:]) + + case sigTypeECDSA: + // Parse the signature with strict checks. + sigBytes := s.ToSignatureBytes() + sig, err := ecdsa.ParseDERSignature(sigBytes) + if err != nil { + return nil, err + } + + return sig, nil + + default: + return nil, fmt.Errorf("unknown sig type: %v", s.sigType) + } +} + +// ToSignatureBytes serializes the target fixed-sized signature into the +// encoding of the primary domain for the signature. For ECDSA signatures, this +// is the raw bytes of a DER encoding. +func (s *Sig) ToSignatureBytes() []byte { + switch s.sigType { + // For ECDSA signatures, we'll convert to DER encoding. + case sigTypeECDSA: + // Extract canonically-padded bigint representations from buffer + r := extractCanonicalPadding(s.bytes[0:32]) + s := extractCanonicalPadding(s.bytes[32:64]) + rLen := uint8(len(r)) + sLen := uint8(len(s)) + + // Create a canonical serialized signature. DER format is: + // 0x30 0x02 r 0x02 s + sigBytes := make([]byte, 6+rLen+sLen) + sigBytes[0] = 0x30 // DER signature magic value + sigBytes[1] = 4 + rLen + sLen // Length of rest of signature + sigBytes[2] = 0x02 // Big integer magic value + sigBytes[3] = rLen // Length of R + sigBytes[rLen+4] = 0x02 // Big integer magic value + sigBytes[rLen+5] = sLen // Length of S + copy(sigBytes[4:], r) // Copy R + copy(sigBytes[rLen+6:], s) // Copy S + + return sigBytes + + // For schnorr signatures, we can use the same internal 64 bytes. + case sigTypeSchnorr: + // We'll make a copy of the signature so we don't return a + // reference into the raw slice. + var sig [64]byte + copy(sig[:], s.bytes[:]) + return sig[:] + + default: + // TODO(roasbeef): can only be called via public methods so + // never reachable? + panic("sig type not set") + } +} + +// extractCanonicalPadding is a utility function to extract the canonical +// padding of a big-endian integer from the wire encoding (a 0-padded +// big-endian integer) such that it passes btcec.canonicalPadding test. +func extractCanonicalPadding(b []byte) []byte { + for i := 0; i < len(b); i++ { + // Found first non-zero byte. + if b[i] > 0 { + // If the MSB is set, we need zero padding. + if b[i]&0x80 == 0x80 { + return append([]byte{0x00}, b[i:]...) + } + return b[i:] + } + } + return []byte{0x00} +} diff --git a/payments/db/migration1/lnwire/typed_fee.go b/payments/db/migration1/lnwire/typed_fee.go new file mode 100644 index 000000000..f9b6c8d01 --- /dev/null +++ b/payments/db/migration1/lnwire/typed_fee.go @@ -0,0 +1,60 @@ +package lnwire + +import ( + "io" + + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + FeeRecordType tlv.Type = 55555 +) + +// Fee represents a fee schedule. +type Fee struct { + BaseFee int32 + FeeRate int32 +} + +// Record returns a TLV record that can be used to encode/decode the fee +// type from a given TLV stream. +func (l *Fee) Record() tlv.Record { + return tlv.MakeStaticRecord( + FeeRecordType, l, 8, feeEncoder, feeDecoder, + ) +} + +// feeEncoder is a custom TLV encoder for the fee record. +func feeEncoder(w io.Writer, val interface{}, buf *[8]byte) error { + v, ok := val.(*Fee) + if !ok { + return tlv.NewTypeForEncodingErr(val, "lnwire.Fee") + } + + if err := tlv.EUint32T(w, uint32(v.BaseFee), buf); err != nil { + return err + } + + return tlv.EUint32T(w, uint32(v.FeeRate), buf) +} + +// feeDecoder is a custom TLV decoder for the fee record. +func feeDecoder(r io.Reader, val interface{}, buf *[8]byte, l uint64) error { + v, ok := val.(*Fee) + if !ok || l != 8 { + return tlv.NewTypeForDecodingErr(val, "lnwire.Fee", l, 8) + } + + var baseFee, feeRate uint32 + if err := tlv.DUint32(r, &baseFee, buf, 4); err != nil { + return err + } + if err := tlv.DUint32(r, &feeRate, buf, 4); err != nil { + return err + } + + v.FeeRate = int32(feeRate) + v.BaseFee = int32(baseFee) + + return nil +} diff --git a/payments/db/migration1/lnwire/update_add_htlc.go b/payments/db/migration1/lnwire/update_add_htlc.go new file mode 100644 index 000000000..38ceeec21 --- /dev/null +++ b/payments/db/migration1/lnwire/update_add_htlc.go @@ -0,0 +1,226 @@ +package lnwire + +import ( + "bytes" + "io" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // OnionPacketSize is the size of the serialized Sphinx onion packet + // included in each UpdateAddHTLC message. The breakdown of the onion + // packet is as follows: 1-byte version, 33-byte ephemeral public key + // (for ECDH), 1300-bytes of per-hop data, and a 32-byte HMAC over the + // entire packet. + OnionPacketSize = 1366 + + // ExperimentalAccountableType is the TLV type used for a custom + // record that sets an experimental accountable value. + ExperimentalAccountableType tlv.Type = 106823 + + // ExperimentalUnaccountable is the value that the experimental + // accountable field contains when a htlc is not accountable. + ExperimentalUnaccountable = 0 + + // ExperimentalAccountable is the value that the experimental + // accountable field contains when a htlc is accountable. We're using a + // single byte to represent our accountable value, but limit the value + // to using the first three bits (max value = 00000111). Interpreted as + // a uint8 (an alias for byte in go), we can just define this constant + // as 7. + ExperimentalAccountable = 7 +) + +type ( + // BlindingPointTlvType is the type for ephemeral pubkeys used in + // route blinding. + BlindingPointTlvType = tlv.TlvType0 + + // BlindingPointRecord holds an optional blinding point on update add + // htlc. + //nolint:ll + BlindingPointRecord = tlv.OptionalRecordT[BlindingPointTlvType, *btcec.PublicKey] +) + +// UpdateAddHTLC is the message sent by Alice to Bob when she wishes to add an +// HTLC to his remote commitment transaction. In addition to information +// detailing the value, the ID, expiry, and the onion blob is also included +// which allows Bob to derive the next hop in the route. The HTLC added by this +// message is to be added to the remote node's "pending" HTLCs. A subsequent +// CommitSig message will move the pending HTLC to the newly created commitment +// transaction, marking them as "staged". +type UpdateAddHTLC struct { + // ChanID is the particular active channel that this UpdateAddHTLC is + // bound to. + ChanID ChannelID + + // ID is the identification server for this HTLC. This value is + // explicitly included as it allows nodes to survive single-sided + // restarts. The ID value for this sides starts at zero, and increases + // with each offered HTLC. + ID uint64 + + // Amount is the amount of millisatoshis this HTLC is worth. + Amount MilliSatoshi + + // PaymentHash is the payment hash to be included in the HTLC this + // request creates. The pre-image to this HTLC must be revealed by the + // upstream peer in order to fully settle the HTLC. + PaymentHash [32]byte + + // Expiry is the number of blocks after which this HTLC should expire. + // It is the receiver's duty to ensure that the outgoing HTLC has a + // sufficient expiry value to allow her to redeem the incoming HTLC. + Expiry uint32 + + // OnionBlob is the raw serialized mix header used to route an HTLC in + // a privacy-preserving manner. The mix header is defined currently to + // be parsed as a 4-tuple: (groupElement, routingInfo, headerMAC, + // body). First the receiving node should use the groupElement, and + // its current onion key to derive a shared secret with the source. + // Once the shared secret has been derived, the headerMAC should be + // checked FIRST. Note that the MAC only covers the routingInfo field. + // If the MAC matches, and the shared secret is fresh, then the node + // should strip off a layer of encryption, exposing the next hop to be + // used in the subsequent UpdateAddHTLC message. + OnionBlob [OnionPacketSize]byte + + // BlindingPoint is the ephemeral pubkey used to optionally blind the + // next hop for this htlc. + BlindingPoint BlindingPointRecord + + // CustomRecords maps TLV types to byte slices, storing arbitrary data + // intended for inclusion in the ExtraData field of the UpdateAddHTLC + // message. + CustomRecords CustomRecords + + // ExtraData is the set of data that was appended to this message to + // fill out the full maximum transport message size. These fields can + // be used to specify optional data such as custom TLV fields. + ExtraData ExtraOpaqueData +} + +// NewUpdateAddHTLC returns a new empty UpdateAddHTLC message. +func NewUpdateAddHTLC() *UpdateAddHTLC { + return &UpdateAddHTLC{} +} + +// A compile time check to ensure UpdateAddHTLC implements the lnwire.Message +// interface. +var _ Message = (*UpdateAddHTLC)(nil) + +// Decode deserializes a serialized UpdateAddHTLC message stored in the passed +// io.Reader observing the specified protocol version. +// +// This is part of the lnwire.Message interface. +func (c *UpdateAddHTLC) Decode(r io.Reader, pver uint32) error { + // msgExtraData is a temporary variable used to read the message extra + // data field from the reader. + var msgExtraData ExtraOpaqueData + + if err := ReadElements(r, + &c.ChanID, + &c.ID, + &c.Amount, + c.PaymentHash[:], + &c.Expiry, + c.OnionBlob[:], + &msgExtraData, + ); err != nil { + return err + } + + // Extract TLV records from the extra data field. + blindingRecord := c.BlindingPoint.Zero() + + customRecords, parsed, extraData, err := ParseAndExtractCustomRecords( + msgExtraData, &blindingRecord, + ) + if err != nil { + return err + } + + // Assign the parsed records back to the message. + if parsed.Contains(blindingRecord.TlvType()) { + c.BlindingPoint = tlv.SomeRecordT(blindingRecord) + } + + c.CustomRecords = customRecords + c.ExtraData = extraData + + return nil +} + +// Encode serializes the target UpdateAddHTLC into the passed io.Writer +// observing the protocol version specified. +// +// This is part of the lnwire.Message interface. +func (c *UpdateAddHTLC) Encode(w *bytes.Buffer, pver uint32) error { + if err := WriteChannelID(w, c.ChanID); err != nil { + return err + } + + if err := WriteUint64(w, c.ID); err != nil { + return err + } + + if err := WriteMilliSatoshi(w, c.Amount); err != nil { + return err + } + + if err := WriteBytes(w, c.PaymentHash[:]); err != nil { + return err + } + + if err := WriteUint32(w, c.Expiry); err != nil { + return err + } + + if err := WriteBytes(w, c.OnionBlob[:]); err != nil { + return err + } + + // Only include blinding point in extra data if present. + var records []tlv.RecordProducer + c.BlindingPoint.WhenSome( + func(b tlv.RecordT[BlindingPointTlvType, *btcec.PublicKey]) { + records = append(records, &b) + }, + ) + + extraData, err := MergeAndEncode(records, c.ExtraData, c.CustomRecords) + if err != nil { + return err + } + + return WriteBytes(w, extraData) +} + +// MsgType returns the integer uniquely identifying this message type on the +// wire. +// +// This is part of the lnwire.Message interface. +func (c *UpdateAddHTLC) MsgType() MessageType { + return MsgUpdateAddHTLC +} + +// TargetChanID returns the channel id of the link for which this message is +// intended. +// +// NOTE: Part of peer.LinkUpdater interface. +func (c *UpdateAddHTLC) TargetChanID() ChannelID { + return c.ChanID +} + +// SerializedSize returns the serialized size of the message in bytes. +// +// This is part of the lnwire.SizeableMessage interface. +func (c *UpdateAddHTLC) SerializedSize() (uint32, error) { + return MessageSerializedSize(c) +} + +// A compile time check to ensure UpdateAddHTLC implements the +// lnwire.SizeableMessage interface. +var _ SizeableMessage = (*UpdateAddHTLC)(nil) diff --git a/payments/db/migration1/lnwire/writer.go b/payments/db/migration1/lnwire/writer.go new file mode 100644 index 000000000..db575663e --- /dev/null +++ b/payments/db/migration1/lnwire/writer.go @@ -0,0 +1,185 @@ +package lnwire + +import ( + "bytes" + "encoding/binary" + "errors" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/btcutil" +) + +var ( + // ErrNilFeatureVector is returned when the supplied feature is nil. + ErrNilFeatureVector = errors.New("cannot write nil feature vector") + + // ErrNilPublicKey is returned when a nil pubkey is used. + ErrNilPublicKey = errors.New("cannot write nil pubkey") +) + +// WriteBytes appends the given bytes to the provided buffer. +func WriteBytes(buf *bytes.Buffer, b []byte) error { + _, err := buf.Write(b) + return err +} + +// WriteUint8 appends the uint8 to the provided buffer. +func WriteUint8(buf *bytes.Buffer, n uint8) error { + _, err := buf.Write([]byte{n}) + return err +} + +// WriteUint16 appends the uint16 to the provided buffer. It encodes the +// integer using big endian byte order. +func WriteUint16(buf *bytes.Buffer, n uint16) error { + var b [2]byte + binary.BigEndian.PutUint16(b[:], n) + _, err := buf.Write(b[:]) + return err +} + +// WriteUint32 appends the uint32 to the provided buffer. It encodes the +// integer using big endian byte order. +func WriteUint32(buf *bytes.Buffer, n uint32) error { + var b [4]byte + binary.BigEndian.PutUint32(b[:], n) + _, err := buf.Write(b[:]) + return err +} + +// WriteUint64 appends the uint64 to the provided buffer. It encodes the +// integer using big endian byte order. +func WriteUint64(buf *bytes.Buffer, n uint64) error { + var b [8]byte + binary.BigEndian.PutUint64(b[:], n) + _, err := buf.Write(b[:]) + return err +} + +// WriteSatoshi appends the Satoshi value to the provided buffer. +func WriteSatoshi(buf *bytes.Buffer, amount btcutil.Amount) error { + return WriteUint64(buf, uint64(amount)) +} + +// WriteMilliSatoshi appends the MilliSatoshi value to the provided buffer. +func WriteMilliSatoshi(buf *bytes.Buffer, amount MilliSatoshi) error { + return WriteUint64(buf, uint64(amount)) +} + +// WritePublicKey appends the compressed public key to the provided buffer. +func WritePublicKey(buf *bytes.Buffer, pub *btcec.PublicKey) error { + if pub == nil { + return ErrNilPublicKey + } + + serializedPubkey := pub.SerializeCompressed() + return WriteBytes(buf, serializedPubkey) +} + +// WriteChannelID appends the ChannelID to the provided buffer. +func WriteChannelID(buf *bytes.Buffer, channelID ChannelID) error { + return WriteBytes(buf, channelID[:]) +} + +// WriteShortChannelID appends the ShortChannelID to the provided buffer. It +// encodes the BlockHeight and TxIndex each using 3 bytes with big endian byte +// order, and encodes txPosition using 2 bytes with big endian byte order. +func WriteShortChannelID(buf *bytes.Buffer, shortChanID ShortChannelID) error { + // Check that field fit in 3 bytes and write the blockHeight + if shortChanID.BlockHeight > ((1 << 24) - 1) { + return errors.New("block height should fit in 3 bytes") + } + + var blockHeight [4]byte + binary.BigEndian.PutUint32(blockHeight[:], shortChanID.BlockHeight) + + if _, err := buf.Write(blockHeight[1:]); err != nil { + return err + } + + // Check that field fit in 3 bytes and write the txIndex + if shortChanID.TxIndex > ((1 << 24) - 1) { + return errors.New("tx index should fit in 3 bytes") + } + + var txIndex [4]byte + binary.BigEndian.PutUint32(txIndex[:], shortChanID.TxIndex) + if _, err := buf.Write(txIndex[1:]); err != nil { + return err + } + + // Write the TxPosition + return WriteUint16(buf, shortChanID.TxPosition) +} + +// WriteSig appends the signature to the provided buffer. +func WriteSig(buf *bytes.Buffer, sig Sig) error { + return WriteBytes(buf, sig.bytes[:]) +} + +// WriteSigs appends the slice of signatures to the provided buffer with its +// length. +func WriteSigs(buf *bytes.Buffer, sigs []Sig) error { + // Write the length of the sigs. + if err := WriteUint16(buf, uint16(len(sigs))); err != nil { + return err + } + + for _, sig := range sigs { + if err := WriteSig(buf, sig); err != nil { + return err + } + } + return nil +} + +// WriteFailCode appends the FailCode to the provided buffer. +func WriteFailCode(buf *bytes.Buffer, e FailCode) error { + return WriteUint16(buf, uint16(e)) +} + +// WriteRawFeatureVector encodes the feature using the feature's Encode method +// and appends the data to the provided buffer. An error will return if the +// passed feature is nil. +func WriteRawFeatureVector(buf *bytes.Buffer, feature *RawFeatureVector) error { + if feature == nil { + return ErrNilFeatureVector + } + + return feature.Encode(buf) +} + +// WriteChanUpdateMsgFlags appends the update flag to the provided buffer. +func WriteChanUpdateMsgFlags(buf *bytes.Buffer, f ChanUpdateMsgFlags) error { + return WriteUint8(buf, uint8(f)) +} + +// WriteChanUpdateChanFlags appends the update flag to the provided buffer. +func WriteChanUpdateChanFlags(buf *bytes.Buffer, f ChanUpdateChanFlags) error { + return WriteUint8(buf, uint8(f)) +} + +// WriteErrorData appends the data to the provided buffer. +func WriteErrorData(buf *bytes.Buffer, data ErrorData) error { + return writeDataWithLength(buf, data) +} + +// WriteBool appends the boolean to the provided buffer. +func WriteBool(buf *bytes.Buffer, b bool) error { + if b { + return WriteBytes(buf, []byte{1}) + } + return WriteBytes(buf, []byte{0}) +} + +// writeDataWithLength writes the data and its length to the buffer. +func writeDataWithLength(buf *bytes.Buffer, data []byte) error { + var l [2]byte + binary.BigEndian.PutUint16(l[:], uint16(len(data))) + if _, err := buf.Write(l[:]); err != nil { + return err + } + + _, err := buf.Write(data) + return err +} diff --git a/payments/db/migration1/log.go b/payments/db/migration1/log.go new file mode 100644 index 000000000..52f1f7505 --- /dev/null +++ b/payments/db/migration1/log.go @@ -0,0 +1,32 @@ +package migration1 + +import ( + "github.com/btcsuite/btclog/v2" + "github.com/lightningnetwork/lnd/build" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log btclog.Logger + +// Subsystem defines the logging identifier for this subsystem. +const Subsystem = "PYDB" + +// The default amount of logging is none. +func init() { + UseLogger(build.NewSubLogger(Subsystem, nil)) +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until UseLogger is called. +func DisableLog() { + UseLogger(btclog.Disabled) +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using btclog. +func UseLogger(logger btclog.Logger) { + log = logger +} diff --git a/payments/db/migration1/migration_external_test.go b/payments/db/migration1/migration_external_test.go new file mode 100644 index 000000000..85f828961 --- /dev/null +++ b/payments/db/migration1/migration_external_test.go @@ -0,0 +1,373 @@ +//go:build test_db_postgres || test_db_sqlite + +package migration1 + +import ( + "context" + "fmt" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/btcsuite/btclog/v2" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/kvdb/postgres" + "github.com/lightningnetwork/lnd/kvdb/sqlbase" + "github.com/lightningnetwork/lnd/kvdb/sqlite" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// TestMigrationWithExternalDB tests the migration of the payment store from a +// bolt backed channel.db or a kvdb channel.sqlite to a SQL database. Note that +// this test does not attempt to be a complete migration test for all payment +// store types but rather is added as a tool for developers and users to debug +// payment migration issues with an actual channel.db/channel.sqlite file. +// +// NOTE: To use this test, place either of those files in the +// payments/db/migration1/testdata directory, uncomment the "Skipf" line, and +// set the "fileName" variable to the name of the channel database file you +// want to use for the migration test. +func TestMigrationWithExternalDB(t *testing.T) { + ctx := context.Background() + + // NOTE: comment this line out to run the test. + t.Skipf("skipping test meant for local debugging only") + + // NOTE: set this to the name of the channel database file you want + // to use for the migration test. This may be either a bbolt ".db" file + // or a SQLite ".sqlite" file. If you want to migrate from a + // bbolt channel.db file, set this to "channel.db". + const fileName = "channel.db" + + // NOTE: if set, this test will prefer migrating from a Postgres-backed + // kvdb source instead of a local file. Leave empty to use fileName. + const postgresKVDSN = "" + const postgresKVPfx = "channeldb" + const logSequenceOrder = false + + // Determine if we are using a SQLite file or a Bolt DB file. + isSqlite := strings.HasSuffix(fileName, ".sqlite") + + // Set up logging for the test. + logger := btclog.NewSLogger(btclog.NewDefaultHandler(os.Stdout)) + UseLogger(logger) + + // migrate runs the migration from the kvdb store to the SQL store + // and then performs a batched deep comparison of every migrated + // payment to ensure all data (including HTLC details) was preserved + // correctly. + migrate := func(t *testing.T, kvBackend kvdb.Backend) { + sqlStore := setupTestSQLDB(t) + + // Run migration in a transaction. + err := sqlStore.db.ExecTx( + ctx, sqldb.WriteTxOpt(), func(tx SQLQueries) error { + migTx, ok := tx.(SQLMigrationQueries) + if !ok { + return fmt.Errorf("db does not " + + "implement SQLMigrationQueries") + } + + return MigratePaymentsKVToSQL( + ctx, kvBackend, migTx, &SQLStoreConfig{ + QueryCfg: sqlStore.cfg.QueryCfg, + }, + ) + }, sqldb.NoOpReset, + ) + require.NoError(t, err) + + t.Log("========================================") + t.Log(" Deep Validation") + t.Log("========================================") + + // Deep compare all migrated payments in batches. + deepValidateAllPayments( + t, ctx, kvBackend, sqlStore, + ) + + _ = logSequenceOrder + } + + connectPostgres := func(t *testing.T, dsn, prefix string) kvdb.Backend { + dsn = strings.TrimSpace(dsn) + if dsn == "" { + t.Fatalf("missing postgres kvdb dsn") + } + + prefix = strings.TrimSpace(prefix) + if prefix == "" { + prefix = "channeldb" + } + + const ( + timeout = 10 * time.Second + maxConns = 5 + ) + sqlbase.Init(maxConns) + + dbCfg := &postgres.Config{ + Dsn: dsn, + Timeout: timeout, + MaxConnections: maxConns, + } + + kvStore, err := kvdb.Open( + kvdb.PostgresBackendName, ctx, dbCfg, prefix, + ) + require.NoError(t, err) + + t.Cleanup(func() { _ = kvStore.Close() }) + + return kvStore + } + + connectPostgresKV := func(t *testing.T) kvdb.Backend { + return connectPostgres(t, postgresKVDSN, postgresKVPfx) + } + + connectBBolt := func(t *testing.T, dbPath string) kvdb.Backend { + cfg := &kvdb.BoltBackendConfig{ + DBPath: dbPath, + DBFileName: fileName, + NoFreelistSync: true, + AutoCompact: false, + AutoCompactMinAge: kvdb.DefaultBoltAutoCompactMinAge, + DBTimeout: kvdb.DefaultDBTimeout, + ReadOnly: true, + } + + kvStore, err := kvdb.GetBoltBackend(cfg) + require.NoError(t, err) + + t.Cleanup(func() { _ = kvStore.Close() }) + + return kvStore + } + + connectSQLite := func(t *testing.T, dbPath string) kvdb.Backend { + const ( + timeout = 10 * time.Second + maxConns = 5 + ) + sqlbase.Init(maxConns) + + cfg := &sqlite.Config{ + Timeout: timeout, + BusyTimeout: timeout, + MaxConnections: maxConns, + } + + kvStore, err := kvdb.Open( + kvdb.SqliteBackendName, ctx, cfg, + dbPath, fileName, + // NOTE: we use the raw string here else we get an + // import cycle if we try to import lncfg.NSChannelDB. + "channeldb", + ) + require.NoError(t, err) + + t.Cleanup(func() { _ = kvStore.Close() }) + + return kvStore + } + + tests := []struct { + name string + dbPath string + }{ + { + name: "testdata", + dbPath: "testdata", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if postgresKVDSN != "" { + migrate(t, connectPostgresKV(t)) + return + } + + chanDBPath := path.Join(test.dbPath, fileName) + t.Logf("Connecting to channel DB at: %s", chanDBPath) + + connectDB := connectBBolt + if isSqlite { + connectDB = connectSQLite + } + + migrate(t, connectDB(t, test.dbPath)) + }) + } +} + +// kvPaymentRef holds a KV payment and its hash for batched deep validation. +type kvPaymentRef struct { + hash lntypes.Hash + payment *MPPayment +} + +// deepValidateAllPayments iterates all KV payments in batches and performs a +// deep comparison against their SQL counterparts. For each batch, it fetches +// the full SQL payment data (including HTLCs, routes, custom records) via +// batch queries and compares field-by-field with the KV data. +func deepValidateAllPayments(t *testing.T, ctx context.Context, + kvBackend kvdb.Backend, sqlStore *SQLStore) { + + t.Helper() + + batchSize := int(sqlStore.cfg.QueryCfg.MaxBatchSize) + + // Get total payment count from SQL so we can show progress. + var totalPayments int64 + err := sqlStore.db.ExecTx( + ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + var err error + totalPayments, err = db.CountPayments(ctx) + return err + }, sqldb.NoOpReset, + ) + require.NoError(t, err) + + var totalValidated int + + // Iterate all KV payments, collecting them into batches. For each + // full batch, perform a batched deep comparison against SQL. + var batch []kvPaymentRef + err = kvdb.View(kvBackend, func(kvTx kvdb.RTx) error { + paymentsBucket := kvTx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + return nil + } + + return paymentsBucket.ForEach(func(k, v []byte) error { + bucket := paymentsBucket.NestedReadBucket(k) + if bucket == nil { + return nil + } + + payment, err := fetchPayment(bucket) + if err != nil { + return err + } + + var hash lntypes.Hash + copy(hash[:], k) + + batch = append(batch, kvPaymentRef{ + hash: hash, + payment: payment, + }) + + if len(batch) >= batchSize { + deepCompareBatch( + t, ctx, sqlStore, batch, + ) + totalValidated += len(batch) + t.Logf("Deep validated %d/%d payments", + totalValidated, totalPayments) + + batch = batch[:0] + } + + return nil + }) + }, func() { + batch = nil + totalValidated = 0 + }) + require.NoError(t, err) + + // Validate any remaining payments in the last batch. + if len(batch) > 0 { + deepCompareBatch(t, ctx, sqlStore, batch) + totalValidated += len(batch) + } + + t.Logf("Deep validated %d/%d payments (complete)", + totalValidated, totalPayments) +} + +// deepCompareBatch performs a batched deep comparison of KV payments against +// their SQL counterparts. It fetches SQL payment data in a single read +// transaction using batch queries. +func deepCompareBatch(t *testing.T, ctx context.Context, + sqlStore *SQLStore, batch []kvPaymentRef) { + + t.Helper() + + err := sqlStore.db.ExecTx( + ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + // Look up each payment by hash to get the SQL row + // with payment ID. + ids := make([]int64, 0, len(batch)) + rowsByHash := make( + map[lntypes.Hash]sqlc.FetchPaymentsByIDsRow, + len(batch), + ) + for _, ref := range batch { + row, err := fetchPaymentByHash( + ctx, db, ref.hash, + ) + if err != nil { + return fmt.Errorf("fetch SQL payment "+ + "%x: %w", ref.hash[:8], err) + } + ids = append(ids, row.Payment.ID) + } + + // Batch-fetch full payment details. + rows, err := db.FetchPaymentsByIDs(ctx, ids) + if err != nil { + return fmt.Errorf("batch fetch payments: %w", + err) + } + for _, row := range rows { + var hash lntypes.Hash + copy(hash[:], row.PaymentIdentifier) + rowsByHash[hash] = row + } + + batchData, err := batchLoadPaymentDetailsData( + ctx, sqlStore.cfg.QueryCfg, db, ids, + ) + if err != nil { + return fmt.Errorf("batch load details: %w", + err) + } + + // Deep compare each payment. + for _, ref := range batch { + row, ok := rowsByHash[ref.hash] + require.True(t, ok, + "SQL payment %x not in batch "+ + "results", ref.hash[:8]) + + sqlPayment, err := buildPaymentFromBatchData( + row, batchData, + ) + require.NoError(t, err, + "build SQL payment %x", + ref.hash[:8]) + + normalizePaymentData(ref.payment) + normalizePaymentData(sqlPayment) + + require.Equal( + t, ref.payment, sqlPayment, + "payment mismatch %x", + ref.hash[:8], + ) + } + + return nil + }, sqldb.NoOpReset, + ) + require.NoError(t, err) +} diff --git a/payments/db/migration1/migration_validation.go b/payments/db/migration1/migration_validation.go new file mode 100644 index 000000000..f20c92783 --- /dev/null +++ b/payments/db/migration1/migration_validation.go @@ -0,0 +1,575 @@ +package migration1 + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "reflect" + "sort" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/record" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/pmezard/go-difflib/difflib" +) + +// migratedPaymentRef is a reference to a migrated payment. +type migratedPaymentRef struct { + Hash lntypes.Hash + PaymentID int64 +} + +// validateMigratedPaymentBatch performs a structural validation pass by +// comparing key fields (hash, amount, fail reason, HTLC count) of KV payments +// with their SQL counterparts. If a structural mismatch is detected, a full +// deep comparison is performed to produce a detailed diff for debugging. +func validateMigratedPaymentBatch(ctx context.Context, + kvBackend kvdb.Backend, sqlDB SQLMigrationQueries, + cfg *SQLStoreConfig, batch []migratedPaymentRef) error { + + if len(batch) == 0 { + return nil + } + + if cfg == nil || cfg.QueryCfg == nil { + return fmt.Errorf("missing SQL store config for validation") + } + + paymentIDs := make([]int64, 0, len(batch)) + for _, item := range batch { + paymentIDs = append(paymentIDs, item.PaymentID) + } + + rows, err := sqlDB.FetchPaymentsByIDsMig(ctx, paymentIDs) + if err != nil { + return fmt.Errorf("fetch SQL payments: %w", err) + } + if len(rows) != len(paymentIDs) { + return fmt.Errorf("SQL payment batch mismatch: got=%d want=%d", + len(rows), len(paymentIDs)) + } + + // Perform the structural check by comparing key fields from the KV + // store with the SQL store. + err = kvBackend.View(func(kvTx kvdb.RTx) error { + paymentsBucket := kvTx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + return fmt.Errorf("no payments bucket") + } + + for _, row := range rows { + hash := row.PaymentIdentifier + var paymentHash lntypes.Hash + copy(paymentHash[:], hash) + + paymentBucket := paymentsBucket.NestedReadBucket(hash) + if paymentBucket == nil { + return fmt.Errorf("missing payment bucket %x", + hash[:8]) + } + + kvPayment, err := fetchPayment(paymentBucket) + if err != nil { + return fmt.Errorf("fetch KV payment %x: %w", + hash[:8], err) + } + + err = structuralCompare(kvPayment, row) + if err != nil { + // On structural mismatch, perform a deep + // comparison to produce a detailed diff. + deepErr := deepComparePayment( + ctx, cfg, sqlDB, row.ID, + paymentHash, kvPayment, + ) + if deepErr != nil { + return deepErr + } + + // If deep comparison passes but structural + // failed, report the structural error as it + // indicates an unexpected inconsistency. + return err + } + + err = compareDuplicatePayments( + ctx, paymentBucket, sqlDB, row.ID, + paymentHash, + ) + if err != nil { + return err + } + } + + return nil + }, func() {}) + if err != nil { + return err + } + + return nil +} + +// structuralCompare performs a fast structural comparison between a KV payment +// and a SQL payment row, checking key fields: payment identifier, amount, +// failure reason, and HTLC attempt count. +func structuralCompare(kvPayment *MPPayment, + sqlRow sqlc.FetchPaymentsByIDsMigRow) error { + + // Compare payment identifier. + kvHash := kvPayment.Info.PaymentIdentifier[:] + if !bytes.Equal(kvHash, sqlRow.PaymentIdentifier) { + return fmt.Errorf("payment identifier mismatch: kv=%x sql=%x", + kvHash[:8], sqlRow.PaymentIdentifier[:8]) + } + + // Compare amount. + kvAmount := int64(kvPayment.Info.Value) + if kvAmount != sqlRow.AmountMsat { + return fmt.Errorf("amount mismatch for %x: kv=%d sql=%d", + sqlRow.PaymentIdentifier[:8], kvAmount, + sqlRow.AmountMsat) + } + + // Compare failure reason. + var kvFailReason sql.NullInt32 + if kvPayment.FailureReason != nil { + kvFailReason = sql.NullInt32{ + Int32: int32(*kvPayment.FailureReason), + Valid: true, + } + } + if kvFailReason != sqlRow.FailReason { + return fmt.Errorf("fail reason mismatch for %x: kv=%v sql=%v", + sqlRow.PaymentIdentifier[:8], kvFailReason, + sqlRow.FailReason) + } + + // Compare HTLC attempt count. + kvHTLCCount := int64(len(kvPayment.HTLCs)) + if kvHTLCCount != sqlRow.HtlcAttemptCount { + return fmt.Errorf("HTLC count mismatch for %x: kv=%d sql=%d", + sqlRow.PaymentIdentifier[:8], kvHTLCCount, + sqlRow.HtlcAttemptCount) + } + + return nil +} + +// deepComparePayment performs a full deep comparison between a KV payment and +// its SQL counterpart, producing a detailed diff on mismatch. +func deepComparePayment(ctx context.Context, cfg *SQLStoreConfig, + sqlDB SQLQueries, paymentID int64, paymentHash lntypes.Hash, + kvPayment *MPPayment) error { + + batchData, err := batchLoadPaymentDetailsData( + ctx, cfg.QueryCfg, sqlDB, []int64{paymentID}, + ) + if err != nil { + return fmt.Errorf("load payment data for deep compare %x: %w", + paymentHash[:8], err) + } + + byIDRows, err := sqlDB.FetchPaymentsByIDs(ctx, []int64{paymentID}) + if err != nil { + return fmt.Errorf("fetch payment by ID for deep compare "+ + "%x: %w", paymentHash[:8], err) + } + if len(byIDRows) != 1 { + return fmt.Errorf("expected 1 payment for deep compare, "+ + "got %d", len(byIDRows)) + } + + sqlPayment, err := buildPaymentFromBatchData(byIDRows[0], batchData) + if err != nil { + return fmt.Errorf("build SQL payment %x: %w", + paymentHash[:8], err) + } + + normalizePaymentForCompare(kvPayment) + normalizePaymentForCompare(sqlPayment) + + if !reflect.DeepEqual(kvPayment, sqlPayment) { + dumpCfg := spew.ConfigState{ + DisablePointerAddresses: true, + DisableCapacities: true, + DisableMethods: true, + SortKeys: true, + } + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines( + dumpCfg.Sdump(kvPayment), + ), + B: difflib.SplitLines( + dumpCfg.Sdump(sqlPayment), + ), + FromFile: "kv", + ToFile: "sql", + Context: 3, + } + diffText, _ := difflib.GetUnifiedDiffString(diff) + + return fmt.Errorf("payment mismatch %x\n%s", + paymentHash[:8], diffText) + } + + return nil +} + +// normalizePaymentForCompare normalizes fields that are expected to differ +// between KV and SQL representations before deep comparison. +func normalizePaymentForCompare(payment *MPPayment) { + if payment == nil { + return + } + + // SequenceNum will not be equal because the kv db can have already + // payments deleted during its lifetime. + payment.SequenceNum = 0 + + // We normalize timestamps before deep-comparing KV vs SQL objects. + // + // - **Microseconds**: SQL backends typically persist timestamps at + // microsecond precision (e.g. Postgres), while KV (Go `time.Time`) + // can contain nanoseconds. Truncating avoids false mismatches caused + // solely by differing storage precision. + // + // - **Local timezone**: when reading from SQL, timestamps are typically + // materialized in the local timezone by the SQL layer (and/or + // converters). Converting both sides to `time.Local` ensures the + // comparison is consistent across KV and SQL representations. + trunc := func(t time.Time) time.Time { + if t.IsZero() { + return t + } + + return time.Unix(0, t.UnixNano()). + In(time.Local). + Truncate(time.Microsecond) + } + + // Normalize PaymentCreationInfo fields. + if payment.Info != nil { + payment.Info.CreationTime = trunc( + payment.Info.CreationTime, + ) + if len(payment.Info.PaymentRequest) == 0 { + payment.Info.PaymentRequest = []byte{} + } + if len(payment.Info.FirstHopCustomRecords) == 0 { + payment.Info.FirstHopCustomRecords = lnwire. + CustomRecords{} + } + } + + // Normalize HTLCAttemptInfo so nil is converted to an empty slice. + if len(payment.HTLCs) == 0 { + payment.HTLCs = []HTLCAttempt{} + } + + // Normalize HTLC attempt ordering; SQL/KV may return attempts + // in different orders. + sort.SliceStable(payment.HTLCs, func(i, j int) bool { + return payment.HTLCs[i].AttemptID < payment.HTLCs[j].AttemptID + }) + + // Normalize HTLCAttemptInfo fields. + for i := range payment.HTLCs { + htlc := &payment.HTLCs[i] + + htlc.AttemptTime = trunc(htlc.AttemptTime) + if htlc.Settle != nil { + htlc.Settle.SettleTime = trunc( + htlc.Settle.SettleTime, + ) + } + if htlc.Failure != nil { + htlc.Failure.FailTime = trunc( + htlc.Failure.FailTime, + ) + } + + // Clear cached fields not persisted in storage. + htlc.onionBlob = [1366]byte{} + htlc.circuit = nil + htlc.cachedSessionKey = nil + + // For legacy payments, the HTLC Hash field may be nil in the + // bbolt backend. During migration, the SQL code uses the + // parent payment hash as fallback. To ensure the comparison + // between bbolt and SQL data succeeds, we apply the same + // fallback here. + // + // See also: patchLegacyPaymentHash in payment_lifecycle.go. + if htlc.Hash == nil && payment.Info != nil { + htlc.Hash = &payment.Info.PaymentIdentifier + } + + if len(htlc.Route.FirstHopWireCustomRecords) == 0 { + htlc.Route.FirstHopWireCustomRecords = + lnwire.CustomRecords{} + } + + for j := range htlc.Route.Hops { + if len(htlc.Route.Hops[j].CustomRecords) == 0 { + htlc.Route.Hops[j].CustomRecords = + record.CustomSet{} + } + + // LegacyPayload was a hint used by the KV store to + // determine how to serialize and deserialize the hop + // payload (i.e. whether to use the legacy format or + // TLV). The SQL store does not serialize hop data at + // all — each field is stored natively in its own + // column — so this flag has no meaning there and is + // never persisted. + htlc.Route.Hops[j].LegacyPayload = false + } + } +} + +// duplicateRecord is a record that represents a duplicate payment. +type duplicateRecord struct { + AmountMsat int64 + CreatedAt time.Time + FailReason sql.NullInt32 + SettlePreimage []byte + SettleTime sql.NullTime +} + +// compareDuplicatePayments validates migrated duplicate rows against KV data. +func compareDuplicatePayments(ctx context.Context, paymentBucket kvdb.RBucket, + sqlDB SQLQueries, paymentID int64, hash lntypes.Hash) error { + + // Fetch the duplicate payments from the KV store. + kvDuplicates, err := fetchDuplicateRecords(paymentBucket) + if err != nil { + return fmt.Errorf("fetch KV duplicates %x: %w", + hash[:8], err) + } + + // Fetch the duplicate payments from the SQL store. + sqlDuplicates, err := sqlDB.FetchPaymentDuplicates(ctx, paymentID) + if err != nil { + return fmt.Errorf("fetch SQL duplicates %x: %w", + hash[:8], err) + } + + if len(kvDuplicates) != len(sqlDuplicates) { + return fmt.Errorf("duplicate count mismatch %x: kv=%d "+ + "sql=%d", hash[:8], len(kvDuplicates), + len(sqlDuplicates)) + } + + kvNormalized := normalizeDuplicateRecords(kvDuplicates) + sqlNormalized := normalizeDuplicateRecords( + dbDuplicatesToDuplicateRecords(sqlDuplicates), + ) + + sortDuplicates(kvNormalized) + sortDuplicates(sqlNormalized) + + if !reflect.DeepEqual(kvNormalized, sqlNormalized) { + dumpCfg := spew.ConfigState{ + DisablePointerAddresses: true, + DisableCapacities: true, + DisableMethods: true, + SortKeys: true, + } + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines( + dumpCfg.Sdump(kvNormalized), + ), + B: difflib.SplitLines( + dumpCfg.Sdump(sqlNormalized), + ), + FromFile: "kv", + ToFile: "sql", + Context: 3, + } + diffText, _ := difflib.GetUnifiedDiffString(diff) + + return fmt.Errorf("duplicate mismatch %x\n%s", + hash[:8], diffText) + } + + return nil +} + +// fetchDuplicateRecords reads duplicate payment records from the KV bucket. +func fetchDuplicateRecords(paymentBucket kvdb.RBucket) ([]duplicateRecord, + error) { + + dupBucket := paymentBucket.NestedReadBucket(duplicatePaymentsBucket) + if dupBucket == nil { + return nil, nil + } + + var duplicates []duplicateRecord + err := dupBucket.ForEach(func(seqBytes, _ []byte) error { + if len(seqBytes) != 8 { + return nil + } + + subBucket := dupBucket.NestedReadBucket(seqBytes) + if subBucket == nil { + return nil + } + + creationData := subBucket.Get(duplicatePaymentCreationInfoKey) + if creationData == nil { + return fmt.Errorf("missing duplicate creation info") + } + + creationInfo, err := deserializeDuplicatePaymentCreationInfo( + bytes.NewReader(creationData), + ) + if err != nil { + return fmt.Errorf("deserialize duplicate creation "+ + "info: %w", err) + } + + settleData := subBucket.Get(duplicatePaymentSettleInfoKey) + failReasonData := subBucket.Get(duplicatePaymentFailInfoKey) + + if settleData != nil && len(failReasonData) > 0 { + return fmt.Errorf("duplicate has both settle and " + + "fail info") + } + + var ( + failReason sql.NullInt32 + settlePreimage []byte + settleTime sql.NullTime + ) + + switch { + case settleData != nil: + settlePreimage, settleTime, err = + parseDuplicateSettleData(settleData) + if err != nil { + return err + } + case len(failReasonData) > 0: + failReason = sql.NullInt32{ + Int32: int32(failReasonData[0]), + Valid: true, + } + default: + // If the duplicate has no settle or fail info, it is + // considered failed. Every duplicate payment must have + // either a settle or fail info in the sql database. So + // we set the fail reason to error to mimic the behavior + // for the kv store. + failReason = sql.NullInt32{ + Int32: int32(FailureReasonError), + Valid: true, + } + } + + duplicates = append(duplicates, duplicateRecord{ + AmountMsat: int64(creationInfo.Value), + CreatedAt: normalizeTimeForSQL( + creationInfo.CreationTime, + ), + FailReason: failReason, + SettlePreimage: settlePreimage, + SettleTime: settleTime, + }) + + return nil + }) + if err != nil { + return nil, err + } + + return duplicates, nil +} + +// dbDuplicatesToDuplicateRecords maps SQL duplicate rows into comparable +// duplicate records. +func dbDuplicatesToDuplicateRecords( + rows []sqlc.PaymentDuplicate) []duplicateRecord { + + duplicates := make([]duplicateRecord, 0, len(rows)) + for _, row := range rows { + duplicates = append(duplicates, duplicateRecord{ + AmountMsat: row.AmountMsat, + CreatedAt: row.CreatedAt, + FailReason: row.FailReason, + SettlePreimage: row.SettlePreimage, + SettleTime: row.SettleTime, + }) + } + + return duplicates +} + +// normalizeDuplicateRecords normalizes time precision and empty fields. +func normalizeDuplicateRecords(records []duplicateRecord) []duplicateRecord { + if len(records) == 0 { + return []duplicateRecord{} + } + + trunc := func(t time.Time) time.Time { + if t.IsZero() { + return t + } + + return t.In(time.Local).Truncate(time.Microsecond) + } + + for i := range records { + records[i].CreatedAt = trunc(records[i].CreatedAt) + if records[i].SettleTime.Valid { + records[i].SettleTime.Time = trunc( + records[i].SettleTime.Time, + ) + } + + if len(records[i].SettlePreimage) == 0 { + records[i].SettlePreimage = []byte{} + } + } + + return records +} + +// sortDuplicates orders records deterministically for deep comparison. +func sortDuplicates(records []duplicateRecord) { + sort.SliceStable(records, func(i, j int) bool { + ai := records[i] + aj := records[j] + + // Duplicates are "duplicates" because they share the same + // payment identifier. So ordering can be stable using + // timestamp + amount. + if !ai.CreatedAt.Equal(aj.CreatedAt) { + return ai.CreatedAt.Before(aj.CreatedAt) + } + + return ai.AmountMsat < aj.AmountMsat + }) +} + +// validatePaymentCounts compares the number of migrated payments with the SQL +// payment count to catch missing rows. +func validatePaymentCounts(ctx context.Context, sqlDB SQLQueries, + expectedCount int64) error { + + sqlCount, err := sqlDB.CountPayments(ctx) + if err != nil { + return fmt.Errorf("count SQL payments: %w", err) + } + if expectedCount != sqlCount { + return fmt.Errorf("payment count mismatch: kv=%d sql=%d", + expectedCount, sqlCount) + } + + return nil +} diff --git a/payments/db/migration1/options.go b/payments/db/migration1/options.go new file mode 100644 index 000000000..382afb26c --- /dev/null +++ b/payments/db/migration1/options.go @@ -0,0 +1,26 @@ +package migration1 + +// StoreOptions holds parameters for the KVStore. +type StoreOptions struct { + // NoMigration allows to open the database in readonly mode + NoMigration bool +} + +// DefaultOptions returns a StoreOptions populated with default values. +func DefaultOptions() *StoreOptions { + return &StoreOptions{ + NoMigration: false, + } +} + +// OptionModifier is a function signature for modifying the default +// StoreOptions. +type OptionModifier func(*StoreOptions) + +// WithNoMigration allows the database to be opened in read only mode by +// disabling migrations. +func WithNoMigration(b bool) OptionModifier { + return func(o *StoreOptions) { + o.NoMigration = b + } +} diff --git a/payments/db/migration1/payment.go b/payments/db/migration1/payment.go new file mode 100644 index 000000000..78599c3ed --- /dev/null +++ b/payments/db/migration1/payment.go @@ -0,0 +1,689 @@ +package migration1 + +import ( + "errors" + "fmt" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + sphinx "github.com/lightningnetwork/lightning-onion" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" +) + +// FailureReason encodes the reason a payment ultimately failed. +type FailureReason byte + +const ( + // FailureReasonTimeout indicates that the payment did timeout before a + // successful payment attempt was made. + FailureReasonTimeout FailureReason = 0 + + // FailureReasonNoRoute indicates no successful route to the + // destination was found during path finding. + FailureReasonNoRoute FailureReason = 1 + + // FailureReasonError indicates that an unexpected error happened during + // payment. + FailureReasonError FailureReason = 2 + + // FailureReasonPaymentDetails indicates that either the hash is unknown + // or the final cltv delta or amount is incorrect. + FailureReasonPaymentDetails FailureReason = 3 + + // FailureReasonInsufficientBalance indicates that we didn't have enough + // balance to complete the payment. + FailureReasonInsufficientBalance FailureReason = 4 + + // FailureReasonCanceled indicates that the payment was canceled by the + // user. + FailureReasonCanceled FailureReason = 5 + + // TODO(joostjager): Add failure reasons for: + // LocalLiquidityInsufficient, RemoteCapacityInsufficient. +) + +// Error returns a human-readable error string for the FailureReason. +func (r FailureReason) Error() string { + return r.String() +} + +// String returns a human-readable FailureReason. +func (r FailureReason) String() string { + switch r { + case FailureReasonTimeout: + return "timeout" + case FailureReasonNoRoute: + return "no_route" + case FailureReasonError: + return "error" + case FailureReasonPaymentDetails: + return "incorrect_payment_details" + case FailureReasonInsufficientBalance: + return "insufficient_balance" + case FailureReasonCanceled: + return "canceled" + } + + return "unknown" +} + +// PaymentCreationInfo is the information necessary to have ready when +// initiating a payment, moving it into state InFlight. +type PaymentCreationInfo struct { + // PaymentIdentifier is the hash this payment is paying to in case of + // non-AMP payments, and the SetID for AMP payments. + PaymentIdentifier lntypes.Hash + + // Value is the amount we are paying. + Value lnwire.MilliSatoshi + + // CreationTime is the time when this payment was initiated. + CreationTime time.Time + + // PaymentRequest is the full payment request, if any. + PaymentRequest []byte + + // FirstHopCustomRecords are the TLV records that are to be sent to the + // first hop of this payment. These records will be transmitted via the + // wire message (UpdateAddHTLC) only and therefore do not affect the + // onion payload size. + FirstHopCustomRecords lnwire.CustomRecords +} + +// String returns a human-readable description of the payment creation info. +func (p *PaymentCreationInfo) String() string { + return fmt.Sprintf("payment_id=%v, amount=%v, created_at=%v", + p.PaymentIdentifier, p.Value, p.CreationTime) +} + +// HTLCAttemptInfo contains static information about a specific HTLC attempt +// for a payment. This information is used by the router to handle any errors +// coming back after an attempt is made, and to query the switch about the +// status of the attempt. +type HTLCAttemptInfo struct { + // AttemptID is the unique ID used for this attempt. + AttemptID uint64 + + // sessionKey is the raw bytes ephemeral key used for this attempt. + // These bytes are lazily read off disk to save ourselves the expensive + // EC operations used by btcec.PrivKeyFromBytes. + sessionKey [btcec.PrivKeyBytesLen]byte + + // cachedSessionKey is our fully deserialized sesionKey. This value + // may be nil if the attempt has just been read from disk and its + // session key has not been used yet. + cachedSessionKey *btcec.PrivateKey + + // Route is the route attempted to send the HTLC. + Route Route + + // AttemptTime is the time at which this HTLC was attempted. + AttemptTime time.Time + + // Hash is the hash used for this single HTLC attempt. For AMP payments + // this will differ across attempts, for non-AMP payments each attempt + // will use the same hash. This can be nil for older payment attempts, + // in which the payment's PaymentHash in the PaymentCreationInfo should + // be used. + Hash *lntypes.Hash + + // onionBlob is the cached value for onion blob created from the sphinx + // construction. + onionBlob [lnwire.OnionPacketSize]byte + + // circuit is the cached value for sphinx circuit. + circuit *sphinx.Circuit +} + +// SessionKey returns the ephemeral key used for a htlc attempt. This function +// performs expensive ec-ops to obtain the session key if it is not cached. +func (h *HTLCAttemptInfo) SessionKey() *btcec.PrivateKey { + if h.cachedSessionKey == nil { + h.cachedSessionKey, _ = btcec.PrivKeyFromBytes( + h.sessionKey[:], + ) + } + + return h.cachedSessionKey +} + +// HTLCAttempt contains information about a specific HTLC attempt for a given +// payment. It contains the HTLCAttemptInfo used to send the HTLC, as well +// as a timestamp and any known outcome of the attempt. +type HTLCAttempt struct { + HTLCAttemptInfo + + // Settle is the preimage of a successful payment. This serves as a + // proof of payment. It will only be non-nil for settled payments. + // + // NOTE: Can be nil if payment is not settled. + Settle *HTLCSettleInfo + + // Fail is a failure reason code indicating the reason the payment + // failed. It is only non-nil for failed payments. + // + // NOTE: Can be nil if payment is not failed. + Failure *HTLCFailInfo +} + +// HTLCSettleInfo encapsulates the information that augments an HTLCAttempt in +// the event that the HTLC is successful. +type HTLCSettleInfo struct { + // Preimage is the preimage of a successful HTLC. This serves as a proof + // of payment. + Preimage lntypes.Preimage + + // SettleTime is the time at which this HTLC was settled. + SettleTime time.Time +} + +// HTLCFailReason is the reason an htlc failed. +type HTLCFailReason byte + +const ( + // HTLCFailUnknown is recorded for htlcs that failed with an unknown + // reason. + HTLCFailUnknown HTLCFailReason = 0 + + // HTLCFailUnreadable is recorded for htlcs that had a failure message + // that couldn't be decrypted. + HTLCFailUnreadable HTLCFailReason = 1 + + // HTLCFailInternal is recorded for htlcs that failed because of an + // internal error. + HTLCFailInternal HTLCFailReason = 2 + + // HTLCFailMessage is recorded for htlcs that failed with a network + // failure message. + HTLCFailMessage HTLCFailReason = 3 +) + +// HTLCFailInfo encapsulates the information that augments an HTLCAttempt in the +// event that the HTLC fails. +type HTLCFailInfo struct { + // FailTime is the time at which this HTLC was failed. + FailTime time.Time + + // Message is the wire message that failed this HTLC. This field will be + // populated when the failure reason is HTLCFailMessage. + Message lnwire.FailureMessage + + // Reason is the failure reason for this HTLC. + Reason HTLCFailReason + + // The position in the path of the intermediate or final node that + // generated the failure message. Position zero is the sender node. This + // field will be populated when the failure reason is either + // HTLCFailMessage or HTLCFailUnknown. + FailureSourceIndex uint32 +} + +// MPPaymentState wraps a series of info needed for a given payment, which is +// used by both MPP and AMP. This is a memory representation of the payment's +// current state and is updated whenever the payment is read from disk. +type MPPaymentState struct { + // NumAttemptsInFlight specifies the number of HTLCs the payment is + // waiting results for. + NumAttemptsInFlight int + + // RemainingAmt specifies how much more money to be sent. + RemainingAmt lnwire.MilliSatoshi + + // FeesPaid specifies the total fees paid so far that can be used to + // calculate remaining fee budget. + FeesPaid lnwire.MilliSatoshi + + // HasSettledHTLC is true if at least one of the payment's HTLCs is + // settled. + HasSettledHTLC bool + + // PaymentFailed is true if the payment has been marked as failed with + // a reason. + PaymentFailed bool +} + +// MPPayment is a wrapper around a payment's PaymentCreationInfo and +// HTLCAttempts. All payments will have the PaymentCreationInfo set, any +// HTLCs made in attempts to be completed will populated in the HTLCs slice. +// Each populated HTLCAttempt represents an attempted HTLC, each of which may +// have the associated Settle or Fail struct populated if the HTLC is no longer +// in-flight. +type MPPayment struct { + // SequenceNum is a unique identifier used to sort the payments in + // order of creation. + SequenceNum uint64 + + // Info holds all static information about this payment, and is + // populated when the payment is initiated. + Info *PaymentCreationInfo + + // HTLCs holds the information about individual HTLCs that we send in + // order to make the payment. + HTLCs []HTLCAttempt + + // FailureReason is the failure reason code indicating the reason the + // payment failed. + // + // NOTE: Will only be set once the daemon has given up on the payment + // altogether. + FailureReason *FailureReason + + // Status is the current PaymentStatus of this payment. + Status PaymentStatus + + // State is the current state of the payment that holds a number of key + // insights and is used to determine what to do on each payment loop + // iteration. + State *MPPaymentState +} + +// Terminated returns a bool to specify whether the payment is in a terminal +// state. +func (m *MPPayment) Terminated() bool { + // If the payment is in terminal state, it cannot be updated. + return m.Status.updatable() != nil +} + +// TerminalInfo returns any HTLC settle info recorded. If no settle info is +// recorded, any payment level failure will be returned. If neither a settle +// nor a failure is recorded, both return values will be nil. +func (m *MPPayment) TerminalInfo() (*HTLCAttempt, *FailureReason) { + for _, h := range m.HTLCs { + if h.Settle != nil { + return &h, nil + } + } + + return nil, m.FailureReason +} + +// SentAmt returns the sum of sent amount and fees for HTLCs that are either +// settled or still in flight. +func (m *MPPayment) SentAmt() (lnwire.MilliSatoshi, lnwire.MilliSatoshi) { + var sent, fees lnwire.MilliSatoshi + for _, h := range m.HTLCs { + if h.Failure != nil { + continue + } + + // The attempt was not failed, meaning the amount was + // potentially sent to the receiver. + sent += h.Route.ReceiverAmt() + fees += h.Route.TotalFees() + } + + return sent, fees +} + +// InFlightHTLCs returns the HTLCs that are still in-flight, meaning they have +// not been settled or failed. +func (m *MPPayment) InFlightHTLCs() []HTLCAttempt { + var inflights []HTLCAttempt + for _, h := range m.HTLCs { + if h.Settle != nil || h.Failure != nil { + continue + } + + inflights = append(inflights, h) + } + + return inflights +} + +// GetAttempt returns the specified htlc attempt on the payment. +func (m *MPPayment) GetAttempt(id uint64) (*HTLCAttempt, error) { + // TODO(yy): iteration can be slow, make it into a tree or use BS. + for _, htlc := range m.HTLCs { + htlc := htlc + if htlc.AttemptID == id { + return &htlc, nil + } + } + + return nil, errors.New("htlc attempt not found on payment") +} + +// Registrable returns an error to specify whether adding more HTLCs to the +// payment with its current status is allowed. A payment can accept new HTLC +// registrations when it's newly created, or none of its HTLCs is in a terminal +// state. +func (m *MPPayment) Registrable() error { + // If updating the payment is not allowed, we can't register new HTLCs. + // Otherwise, the status must be either `StatusInitiated` or + // `StatusInFlight`. + if err := m.Status.updatable(); err != nil { + return err + } + + // Exit early if this is not inflight. + if m.Status != StatusInFlight { + return nil + } + + // There are still inflight HTLCs and we need to check whether there + // are settled HTLCs or the payment is failed. If we already have + // settled HTLCs, we won't allow adding more HTLCs. + if m.State.HasSettledHTLC { + return ErrPaymentPendingSettled + } + + // If the payment is already failed, we won't allow adding more HTLCs. + if m.State.PaymentFailed { + return ErrPaymentPendingFailed + } + + // Otherwise we can add more HTLCs. + return nil +} + +// setState creates and attaches a new MPPaymentState to the payment. It also +// updates the payment's status based on its current state. +func (m *MPPayment) setState() error { + // Fetch the total amount and fees that has already been sent in + // settled and still in-flight shards. + sentAmt, fees := m.SentAmt() + + // Sanity check we haven't sent a value larger than the payment amount. + totalAmt := m.Info.Value + if sentAmt > totalAmt { + return fmt.Errorf("%w: sent=%v, total=%v", + ErrSentExceedsTotal, sentAmt, totalAmt) + } + + // Get any terminal info for this payment. + settle, failure := m.TerminalInfo() + + // Now determine the payment's status. + status, err := decidePaymentStatus(m.HTLCs, m.FailureReason) + if err != nil { + return err + } + + // Update the payment state and status. + m.State = &MPPaymentState{ + NumAttemptsInFlight: len(m.InFlightHTLCs()), + RemainingAmt: totalAmt - sentAmt, + FeesPaid: fees, + HasSettledHTLC: settle != nil, + PaymentFailed: failure != nil, + } + m.Status = status + + return nil +} + +// SetState calls the internal method setState. This is a temporary method +// to be used by the tests in routing. Once the tests are updated to use mocks, +// this method can be removed. +// +// TODO(yy): delete. +func (m *MPPayment) SetState() error { + return m.setState() +} + +// NeedWaitAttempts decides whether we need to hold creating more HTLC attempts +// and wait for the results of the payment's inflight HTLCs. Return an error if +// the payment is in an unexpected state. +func (m *MPPayment) NeedWaitAttempts() (bool, error) { + // Check when the remainingAmt is not zero, which means we have more + // money to be sent. + if m.State.RemainingAmt != 0 { + switch m.Status { + // If the payment is newly created, no need to wait for HTLC + // results. + case StatusInitiated: + return false, nil + + // If we have inflight HTLCs, we'll check if we have terminal + // states to decide if we need to wait. + case StatusInFlight: + // We still have money to send, and one of the HTLCs is + // settled. We'd stop sending money and wait for all + // inflight HTLC attempts to finish. + if m.State.HasSettledHTLC { + log.Warnf("payment=%v has remaining amount "+ + "%v, yet at least one of its HTLCs is "+ + "settled", m.Info.PaymentIdentifier, + m.State.RemainingAmt) + + return true, nil + } + + // The payment has a failure reason though we still + // have money to send, we'd stop sending money and wait + // for all inflight HTLC attempts to finish. + if m.State.PaymentFailed { + return true, nil + } + + // Otherwise we don't need to wait for inflight HTLCs + // since we still have money to be sent. + return false, nil + + // We need to send more money, yet the payment is already + // succeeded. Return an error in this case as the receiver is + // violating the protocol. + case StatusSucceeded: + return false, fmt.Errorf("%w: parts of the payment "+ + "already succeeded but still have remaining "+ + "amount %v", ErrPaymentInternal, + m.State.RemainingAmt) + + // The payment is failed and we have no inflight HTLCs, no need + // to wait. + case StatusFailed: + return false, nil + + // Unknown payment status. + default: + return false, fmt.Errorf("%w: %s", + ErrUnknownPaymentStatus, m.Status) + } + } + + // Now we determine whether we need to wait when the remainingAmt is + // already zero. + switch m.Status { + // When the payment is newly created, yet the payment has no remaining + // amount, return an error. + case StatusInitiated: + return false, fmt.Errorf("%w: %v", + ErrPaymentInternal, m.Status) + + // If the payment is inflight, we must wait. + // + // NOTE: an edge case is when all HTLCs are failed while the payment is + // not failed we'd still be in this inflight state. However, since the + // remainingAmt is zero here, it means we cannot be in that state as + // otherwise the remainingAmt would not be zero. + case StatusInFlight: + return true, nil + + // If the payment is already succeeded, no need to wait. + case StatusSucceeded: + return false, nil + + // If the payment is already failed, yet the remaining amount is zero, + // return an error as this indicates an error state. We will only each + // this status when there are no inflight HTLCs and the payment is + // marked as failed with a reason, which means the remainingAmt must + // not be zero because our sentAmt is zero. + case StatusFailed: + return false, fmt.Errorf("%w: %v", + ErrPaymentInternal, m.Status) + + // Unknown payment status. + default: + return false, fmt.Errorf("%w: %s", + ErrUnknownPaymentStatus, m.Status) + } +} + +// GetState returns the internal state of the payment. +func (m *MPPayment) GetState() *MPPaymentState { + return m.State +} + +// GetStatus returns the current status of the payment. +func (m *MPPayment) GetStatus() PaymentStatus { + return m.Status +} + +// GetHTLCs returns all the HTLCs for this payment. +func (m *MPPayment) GetHTLCs() []HTLCAttempt { + return m.HTLCs +} + +// AllowMoreAttempts is used to decide whether we can safely attempt more HTLCs +// for a given payment state. Return an error if the payment is in an +// unexpected state. +func (m *MPPayment) AllowMoreAttempts() (bool, error) { + // Now check whether the remainingAmt is zero or not. If we don't have + // any remainingAmt, no more HTLCs should be made. + if m.State.RemainingAmt == 0 { + // If the payment is newly created, yet we don't have any + // remainingAmt, return an error. + if m.Status == StatusInitiated { + return false, fmt.Errorf("%w: initiated payment has "+ + "zero remainingAmt", + ErrPaymentInternal) + } + + // Otherwise, exit early since all other statuses with zero + // remainingAmt indicate no more HTLCs can be made. + return false, nil + } + + // Otherwise, the remaining amount is not zero, we now decide whether + // to make more attempts based on the payment's current status. + // + // If at least one of the payment's attempts is settled, yet we haven't + // sent all the amount, it indicates something is wrong with the peer + // as the preimage is received. In this case, return an error state. + if m.Status == StatusSucceeded { + return false, fmt.Errorf("%w: payment already succeeded but "+ + "still have remaining amount %v", + ErrPaymentInternal, m.State.RemainingAmt) + } + + // Now check if we can register a new HTLC. + err := m.Registrable() + if err != nil { + log.Warnf("Payment(%v): cannot register HTLC attempt: %v, "+ + "current status: %s", m.Info.PaymentIdentifier, + err, m.Status) + + return false, nil + } + + // Now we know we can register new HTLCs. + return true, nil +} + +// verifyAttempt validates that a new HTLC attempt is compatible with the +// existing payment and its in-flight HTLCs. This function checks: +// 1. MPP (Multi-Path Payment) compatibility between attempts +// 2. Blinded payment consistency +// 3. Amount validation +// 4. Total payment amount limits +func verifyAttempt(payment *MPPayment, attempt *HTLCAttemptInfo) error { + // If the final hop has encrypted data, then we know this is a + // blinded payment. In blinded payments, MPP records are not set + // for split payments and the recipient is responsible for using + // a consistent PathID across the various encrypted data + // payloads that we received from them for this payment. All we + // need to check is that the total amount field for each HTLC + // in the split payment is correct. + isBlinded := len(attempt.Route.FinalHop().EncryptedData) != 0 + + // For blinded payments, the last hop must set the total amount. + if isBlinded { + if attempt.Route.FinalHop().TotalAmtMsat == 0 { + return ErrBlindedPaymentMissingTotalAmount + } + } + + // Make sure any existing shards match the new one with regards + // to MPP options. + mpp := attempt.Route.FinalHop().MPP + + // MPP records should not be set for attempts to blinded paths. + if isBlinded && mpp != nil { + return ErrMPPRecordInBlindedPayment + } + + for _, h := range payment.InFlightHTLCs() { + hMpp := h.Route.FinalHop().MPP + hBlinded := len(h.Route.FinalHop().EncryptedData) != 0 + + // If this is a blinded payment, then no existing HTLCs + // should have MPP records. + if isBlinded && hMpp != nil { + return ErrMPPRecordInBlindedPayment + } + + // If the payment is blinded (previous attempts used blinded + // paths) and the attempt is not, or vice versa, return an + // error. + if isBlinded != hBlinded { + return ErrMixedBlindedAndNonBlindedPayments + } + + // If this is a blinded payment, then we just need to + // check that the TotalAmtMsat field for this shard + // is equal to that of any other shard in the same + // payment. + if isBlinded { + if attempt.Route.FinalHop().TotalAmtMsat != + h.Route.FinalHop().TotalAmtMsat { + + return ErrBlindedPaymentTotalAmountMismatch + } + + continue + } + + switch { + // We tried to register a non-MPP attempt for a MPP + // payment. + case mpp == nil && hMpp != nil: + return ErrMPPayment + + // We tried to register a MPP shard for a non-MPP + // payment. + case mpp != nil && hMpp == nil: + return ErrNonMPPayment + + // Non-MPP payment, nothing more to validate. + case mpp == nil: + continue + } + + // Check that MPP options match. + if mpp.PaymentAddr() != hMpp.PaymentAddr() { + return ErrMPPPaymentAddrMismatch + } + + if mpp.TotalMsat() != hMpp.TotalMsat() { + return ErrMPPTotalAmountMismatch + } + } + + // If this is a non-MPP attempt, it must match the total amount + // exactly. Note that a blinded payment is considered an MPP + // attempt. + amt := attempt.Route.ReceiverAmt() + if !isBlinded && mpp == nil && amt != payment.Info.Value { + return ErrValueMismatch + } + + // Ensure we aren't sending more than the total payment amount. + sentAmt, _ := payment.SentAmt() + if sentAmt+amt > payment.Info.Value { + return fmt.Errorf("%w: attempted=%v, payment amount=%v", + ErrValueExceedsAmt, sentAmt+amt, payment.Info.Value) + } + + return nil +} diff --git a/payments/db/migration1/payment_status.go b/payments/db/migration1/payment_status.go new file mode 100644 index 000000000..16c4b90fb --- /dev/null +++ b/payments/db/migration1/payment_status.go @@ -0,0 +1,257 @@ +package migration1 + +import ( + "fmt" +) + +// PaymentStatus represent current status of payment. +type PaymentStatus byte + +const ( + // NOTE: PaymentStatus = 0 was previously used for status unknown and + // is now deprecated. + + // StatusInitiated is the status where a payment has just been + // initiated. + StatusInitiated PaymentStatus = 1 + + // StatusInFlight is the status where a payment has been initiated, but + // a response has not been received. + StatusInFlight PaymentStatus = 2 + + // StatusSucceeded is the status where a payment has been initiated and + // the payment was completed successfully. + StatusSucceeded PaymentStatus = 3 + + // StatusFailed is the status where a payment has been initiated and a + // failure result has come back. + StatusFailed PaymentStatus = 4 +) + +// errPaymentStatusUnknown is returned when a payment has an unknown status. +var errPaymentStatusUnknown = fmt.Errorf("unknown payment status") + +// String returns readable representation of payment status. +func (ps PaymentStatus) String() string { + switch ps { + case StatusInitiated: + return "Initiated" + + case StatusInFlight: + return "In Flight" + + case StatusSucceeded: + return "Succeeded" + + case StatusFailed: + return "Failed" + + default: + return "Unknown" + } +} + +// initializable returns an error to specify whether initiating the payment +// with its current status is allowed. A payment can only be initialized if it +// hasn't been created yet or already failed. +func (ps PaymentStatus) initializable() error { + switch ps { + // The payment has been created already. We will disallow creating it + // again in case other goroutines have already been creating HTLCs for + // it. + case StatusInitiated: + return ErrPaymentExists + + // We already have an InFlight payment on the network. We will disallow + // any new payments. + case StatusInFlight: + return ErrPaymentInFlight + + // The payment has been attempted and is succeeded so we won't allow + // creating it again. + case StatusSucceeded: + return ErrAlreadyPaid + + // We allow retrying failed payments. + case StatusFailed: + return nil + + default: + return fmt.Errorf("%w: %v", ErrUnknownPaymentStatus, + ps) + } +} + +// removable returns an error to specify whether deleting the payment with its +// current status is allowed. A payment cannot be safely deleted if it has +// inflight HTLCs. +func (ps PaymentStatus) removable() error { + switch ps { + // The payment has been created but has no HTLCs and can be removed. + case StatusInitiated: + return nil + + // There are still inflight HTLCs and the payment needs to wait for the + // final outcomes. + case StatusInFlight: + return ErrPaymentInFlight + + // The payment has been attempted and is succeeded and is allowed to be + // removed. + case StatusSucceeded: + return nil + + // Failed payments are allowed to be removed. + case StatusFailed: + return nil + + default: + return fmt.Errorf("%w: %v", ErrUnknownPaymentStatus, + ps) + } +} + +// updatable returns an error to specify whether the payment's HTLCs can be +// updated. A payment can update its HTLCs when it has inflight HTLCs. +func (ps PaymentStatus) updatable() error { + switch ps { + // Newly created payments can be updated. + case StatusInitiated: + return nil + + // Inflight payments can be updated. + case StatusInFlight: + return nil + + // If the payment has a terminal condition, we won't allow any updates. + case StatusSucceeded: + return ErrPaymentAlreadySucceeded + + case StatusFailed: + return ErrPaymentAlreadyFailed + + default: + return fmt.Errorf("%w: %v", ErrUnknownPaymentStatus, + ps) + } +} + +// decidePaymentStatus uses the payment's DB state to determine a memory status +// that's used by the payment router to decide following actions. +// Together, we use four variables to determine the payment's status, +// - inflight: whether there are any pending HTLCs. +// - settled: whether any of the HTLCs has been settled. +// - htlc failed: whether any of the HTLCs has been failed. +// - payment failed: whether the payment has been marked as failed. +// +// Based on the above variables, we derive the status using the following +// table, +// | inflight | settled | htlc failed | payment failed | status | +// |:--------:|:-------:|:-----------:|:--------------:|:--------------------:| +// | true | true | true | true | StatusInFlight | +// | true | true | true | false | StatusInFlight | +// | true | true | false | true | StatusInFlight | +// | true | true | false | false | StatusInFlight | +// | true | false | true | true | StatusInFlight | +// | true | false | true | false | StatusInFlight | +// | true | false | false | true | StatusInFlight | +// | true | false | false | false | StatusInFlight | +// | false | true | true | true | StatusSucceeded | +// | false | true | true | false | StatusSucceeded | +// | false | true | false | true | StatusSucceeded | +// | false | true | false | false | StatusSucceeded | +// | false | false | true | true | StatusFailed | +// | false | false | true | false | StatusInFlight | +// | false | false | false | true | StatusFailed | +// | false | false | false | false | StatusInitiated | +// +// When `inflight`, `settled`, `htlc failed`, and `payment failed` are false, +// this indicates the payment is newly created and hasn't made any HTLCs yet. +// When `inflight` and `settled` are false, `htlc failed` is true yet `payment +// failed` is false, this indicates all the payment's HTLCs have occurred a +// temporarily failure and the payment is still in-flight. +func decidePaymentStatus(htlcs []HTLCAttempt, + reason *FailureReason) (PaymentStatus, error) { + + var ( + inflight bool + htlcSettled bool + htlcFailed bool + paymentFailed bool + ) + + // If we have a failure reason, the payment is failed. + if reason != nil { + paymentFailed = true + } + + // Go through all HTLCs for this payment, check whether we have any + // settled HTLC, and any still in-flight. + for _, h := range htlcs { + if h.Failure != nil { + htlcFailed = true + continue + } + + if h.Settle != nil { + htlcSettled = true + continue + } + + // If any of the HTLCs are not failed nor settled, we + // still have inflight HTLCs. + inflight = true + } + + // Use the DB state to determine the status of the payment. + switch { + // If we have inflight HTLCs, no matter we have settled or failed + // HTLCs, or the payment failed, we still consider it inflight so we + // inform upper systems to wait for the results. + case inflight: + return StatusInFlight, nil + + // If we have no in-flight HTLCs, and at least one of the HTLCs is + // settled, the payment succeeded. + // + // NOTE: when reaching this case, paymentFailed could be true, which + // means we have a conflicting state for this payment. We choose to + // mark the payment as succeeded because it's the receiver's + // responsibility to only settle the payment iff all HTLCs are + // received. + case htlcSettled: + return StatusSucceeded, nil + + // If we have no in-flight HTLCs, and the payment failure is set, the + // payment is considered failed. + // + // NOTE: when reaching this case, settled must be false. + case paymentFailed: + return StatusFailed, nil + + // If we have no in-flight HTLCs, yet the payment is NOT failed, it + // means all the HTLCs are failed. In this case we can attempt more + // HTLCs. + // + // NOTE: when reaching this case, both settled and paymentFailed must + // be false. + case htlcFailed: + return StatusInFlight, nil + + // If none of the HTLCs is either settled or failed, and we have no + // inflight HTLCs, this means the payment has no HTLCs created yet. + // + // NOTE: when reaching this case, both settled and paymentFailed must + // be false. + case !htlcFailed: + return StatusInitiated, nil + + // Otherwise an impossible state is reached. + // + // NOTE: we should never end up here. + default: + log.Error("Impossible payment state reached") + return 0, fmt.Errorf("%w: payment is corrupted", + errPaymentStatusUnknown) + } +} diff --git a/payments/db/migration1/query.go b/payments/db/migration1/query.go new file mode 100644 index 000000000..1fab2fbd9 --- /dev/null +++ b/payments/db/migration1/query.go @@ -0,0 +1,75 @@ +package migration1 + +const ( + // DefaultMaxPayments is the default maximum number of payments returned + // in the payments query pagination. + DefaultMaxPayments = 100 +) + +// Query represents a query to the payments database starting or ending +// at a certain offset index. The number of retrieved records can be limited. +type Query struct { + // IndexOffset determines the starting point of the payments query and + // is always exclusive. In normal order, the query starts at the next + // higher (available) index compared to IndexOffset. In reversed order, + // the query ends at the next lower (available) index compared to the + // IndexOffset. In the case of a zero index_offset, the query will start + // with the oldest payment when paginating forwards, or will end with + // the most recent payment when paginating backwards. + IndexOffset uint64 + + // MaxPayments is the maximal number of payments returned in the + // payments query. + MaxPayments uint64 + + // Reversed gives a meaning to the IndexOffset. If reversed is set to + // true, the query will fetch payments with indices lower than the + // IndexOffset, otherwise, it will return payments with indices greater + // than the IndexOffset. + Reversed bool + + // If IncludeIncomplete is true, then return payments that have not yet + // fully completed. This means that pending payments, as well as failed + // payments will show up if this field is set to true. + IncludeIncomplete bool + + // CountTotal indicates that all payments currently present in the + // payment index (complete and incomplete) should be counted. + CountTotal bool + + // CreationDateStart, expressed in Unix seconds, if set, filters out + // all payments with a creation date greater than or equal to it. + CreationDateStart int64 + + // CreationDateEnd, expressed in Unix seconds, if set, filters out all + // payments with a creation date less than or equal to it. + CreationDateEnd int64 +} + +// Response contains the result of a query to the payments database. +// It includes the set of payments that match the query and integers which +// represent the index of the first and last item returned in the series of +// payments. These integers allow callers to resume their query in the event +// that the query's response exceeds the max number of returnable events. +type Response struct { + // Payments is the set of payments returned from the database for the + // Query. + Payments []*MPPayment + + // FirstIndexOffset is the index of the first element in the set of + // returned MPPayments. Callers can use this to resume their query + // in the event that the slice has too many events to fit into a single + // response. The offset can be used to continue reverse pagination. + FirstIndexOffset uint64 + + // LastIndexOffset is the index of the last element in the set of + // returned MPPayments. Callers can use this to resume their query + // in the event that the slice has too many events to fit into a single + // response. The offset can be used to continue forward pagination. + LastIndexOffset uint64 + + // TotalCount represents the total number of payments that are currently + // stored in the payment database. This will only be set if the + // CountTotal field in the query was set to true. + TotalCount uint64 +} diff --git a/payments/db/migration1/record/amp.go b/payments/db/migration1/record/amp.go new file mode 100644 index 000000000..f63c7a141 --- /dev/null +++ b/payments/db/migration1/record/amp.go @@ -0,0 +1,121 @@ +package record + +import ( + "fmt" + "io" + + "github.com/lightningnetwork/lnd/tlv" +) + +// AMPOnionType is the type used in the onion to reference the AMP fields: +// root_share, set_id, and child_index. +const AMPOnionType tlv.Type = 14 + +// AMP is a record that encodes the fields necessary for atomic multi-path +// payments. +type AMP struct { + rootShare [32]byte + setID [32]byte + childIndex uint32 +} + +// MaxAmpPayLoadSize is an AMP Record which when serialized to a tlv record uses +// the maximum payload size. The `childIndex` is created randomly and is a +// 4 byte `varint` type so we make sure we use an index which will be encoded in +// 4 bytes. +var MaxAmpPayLoadSize = AMP{ + rootShare: [32]byte{}, + setID: [32]byte{}, + childIndex: 0x80000000, +} + +// NewAMP generate a new AMP record with the given root_share, set_id, and +// child_index. +func NewAMP(rootShare, setID [32]byte, childIndex uint32) *AMP { + return &{ + rootShare: rootShare, + setID: setID, + childIndex: childIndex, + } +} + +// RootShare returns the root share contained in the AMP record. +func (a *AMP) RootShare() [32]byte { + return a.rootShare +} + +// SetID returns the set id contained in the AMP record. +func (a *AMP) SetID() [32]byte { + return a.setID +} + +// ChildIndex returns the child index contained in the AMP record. +func (a *AMP) ChildIndex() uint32 { + return a.childIndex +} + +// AMPEncoder writes the AMP record to the provided io.Writer. +func AMPEncoder(w io.Writer, val interface{}, buf *[8]byte) error { + if v, ok := val.(*AMP); ok { + if err := tlv.EBytes32(w, &v.rootShare, buf); err != nil { + return err + } + + if err := tlv.EBytes32(w, &v.setID, buf); err != nil { + return err + } + + return tlv.ETUint32T(w, v.childIndex, buf) + } + return tlv.NewTypeForEncodingErr(val, "AMP") +} + +const ( + // minAMPLength is the minimum length of a serialized AMP TLV record, + // which occurs when the truncated encoding of child_index takes 0 + // bytes, leaving only the root_share and set_id. + minAMPLength = 64 + + // maxAMPLength is the maximum length of a serialized AMP TLV record, + // which occurs when the truncated encoding of a child_index takes 2 + // bytes. + maxAMPLength = 68 +) + +// AMPDecoder reads the AMP record from the provided io.Reader. +func AMPDecoder(r io.Reader, val interface{}, buf *[8]byte, l uint64) error { + if v, ok := val.(*AMP); ok && minAMPLength <= l && l <= maxAMPLength { + if err := tlv.DBytes32(r, &v.rootShare, buf, 32); err != nil { + return err + } + + if err := tlv.DBytes32(r, &v.setID, buf, 32); err != nil { + return err + } + + return tlv.DTUint32(r, &v.childIndex, buf, l-minAMPLength) + } + return tlv.NewTypeForDecodingErr(val, "AMP", l, maxAMPLength) +} + +// Record returns a tlv.Record that can be used to encode or decode this record. +func (a *AMP) Record() tlv.Record { + return tlv.MakeDynamicRecord( + AMPOnionType, a, a.PayloadSize, AMPEncoder, AMPDecoder, + ) +} + +// PayloadSize returns the size this record takes up in encoded form. +func (a *AMP) PayloadSize() uint64 { + return 32 + 32 + tlv.SizeTUint32(a.childIndex) +} + +// String returns a human-readable description of the amp payload fields. +func (a *AMP) String() string { + if a == nil { + return "" + } + + return fmt.Sprintf("root_share=%x set_id=%x child_index=%d", + a.rootShare, a.setID, a.childIndex) +} diff --git a/payments/db/migration1/record/blinded_data.go b/payments/db/migration1/record/blinded_data.go new file mode 100644 index 000000000..22c09672f --- /dev/null +++ b/payments/db/migration1/record/blinded_data.go @@ -0,0 +1,440 @@ +package record + +import ( + "bytes" + "encoding/binary" + "io" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/tlv" +) + +// AverageDummyHopPayloadSize is the size of a standard blinded path dummy hop +// payload. In most cases, this is larger than the other payload types and so +// to make sure that a sender cannot use this fact to know if a dummy hop is +// present or not, we'll make sure to always pad all payloads to at least this +// size. +const AverageDummyHopPayloadSize = 51 + +// BlindedRouteData contains the information that is included in a blinded +// route encrypted data blob that is created by the recipient to provide +// forwarding information. +type BlindedRouteData struct { + // Padding is an optional set of bytes that a recipient can use to pad + // the data so that the encrypted recipient data blobs are all the same + // length. + Padding tlv.OptionalRecordT[tlv.TlvType1, []byte] + + // ShortChannelID is the channel ID of the next hop. + ShortChannelID tlv.OptionalRecordT[tlv.TlvType2, lnwire.ShortChannelID] + + // NextNodeID is the node ID of the next node on the path. In the + // context of blinded path payments, this is used to indicate the + // presence of dummy hops that need to be peeled from the onion. + NextNodeID tlv.OptionalRecordT[tlv.TlvType4, *btcec.PublicKey] + + // PathID is a secret set of bytes that the blinded path creator will + // set so that they can check the value on decryption to ensure that the + // path they created was used for the intended purpose. + PathID tlv.OptionalRecordT[tlv.TlvType6, []byte] + + // NextBlindingOverride is a blinding point that should be switched + // in for the next hop. This is used to combine two blinded paths into + // one (which primarily is used in onion messaging, but in theory + // could be used for payments as well). + NextBlindingOverride tlv.OptionalRecordT[tlv.TlvType8, *btcec.PublicKey] + + // RelayInfo provides the relay parameters for the hop. + RelayInfo tlv.OptionalRecordT[tlv.TlvType10, PaymentRelayInfo] + + // Constraints provides the payment relay constraints for the hop. + Constraints tlv.OptionalRecordT[tlv.TlvType12, PaymentConstraints] + + // Features is the set of features the payment requires. + Features tlv.OptionalRecordT[tlv.TlvType14, lnwire.FeatureVector] +} + +// NewNonFinalBlindedRouteData creates the data that's provided for hops within +// a blinded route. +func NewNonFinalBlindedRouteData(chanID lnwire.ShortChannelID, + blindingOverride *btcec.PublicKey, relayInfo PaymentRelayInfo, + constraints *PaymentConstraints, + features *lnwire.FeatureVector) *BlindedRouteData { + + info := &BlindedRouteData{ + ShortChannelID: tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType2](chanID), + ), + RelayInfo: tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType10](relayInfo), + ), + } + + if blindingOverride != nil { + info.NextBlindingOverride = tlv.SomeRecordT( + tlv.NewPrimitiveRecord[tlv.TlvType8](blindingOverride)) + } + + if constraints != nil { + info.Constraints = tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType12](*constraints)) + } + + if features != nil { + info.Features = tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType14](*features), + ) + } + + return info +} + +// NewFinalHopBlindedRouteData creates the data that's provided for the final +// hop in a blinded route. +func NewFinalHopBlindedRouteData(constraints *PaymentConstraints, + pathID []byte) *BlindedRouteData { + + var data BlindedRouteData + if pathID != nil { + data.PathID = tlv.SomeRecordT( + tlv.NewPrimitiveRecord[tlv.TlvType6](pathID), + ) + } + + if constraints != nil { + data.Constraints = tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType12](*constraints)) + } + + return &data +} + +// NewDummyHopRouteData creates the data that's provided for any hop preceding +// a dummy hop. The presence of such a payload indicates to the reader that +// they are the intended recipient and should peel the remainder of the onion. +func NewDummyHopRouteData(ourPubKey *btcec.PublicKey, + relayInfo PaymentRelayInfo, + constraints PaymentConstraints) *BlindedRouteData { + + return &BlindedRouteData{ + NextNodeID: tlv.SomeRecordT( + tlv.NewPrimitiveRecord[tlv.TlvType4](ourPubKey), + ), + RelayInfo: tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType10](relayInfo), + ), + Constraints: tlv.SomeRecordT( + tlv.NewRecordT[tlv.TlvType12](constraints), + ), + } +} + +// DecodeBlindedRouteData decodes the data provided within a blinded route. +func DecodeBlindedRouteData(r io.Reader) (*BlindedRouteData, error) { + var ( + d BlindedRouteData + + padding = d.Padding.Zero() + scid = d.ShortChannelID.Zero() + nextNodeID = d.NextNodeID.Zero() + pathID = d.PathID.Zero() + blindingOverride = d.NextBlindingOverride.Zero() + relayInfo = d.RelayInfo.Zero() + constraints = d.Constraints.Zero() + features = d.Features.Zero() + ) + + var tlvRecords lnwire.ExtraOpaqueData + if err := lnwire.ReadElements(r, &tlvRecords); err != nil { + return nil, err + } + + typeMap, err := tlvRecords.ExtractRecords( + &padding, &scid, &nextNodeID, &pathID, &blindingOverride, + &relayInfo, &constraints, &features, + ) + if err != nil { + return nil, err + } + + val, ok := typeMap[d.Padding.TlvType()] + if ok && val == nil { + d.Padding = tlv.SomeRecordT(padding) + } + + if val, ok := typeMap[d.ShortChannelID.TlvType()]; ok && val == nil { + d.ShortChannelID = tlv.SomeRecordT(scid) + } + + if val, ok := typeMap[d.NextNodeID.TlvType()]; ok && val == nil { + d.NextNodeID = tlv.SomeRecordT(nextNodeID) + } + + if val, ok := typeMap[d.PathID.TlvType()]; ok && val == nil { + d.PathID = tlv.SomeRecordT(pathID) + } + + val, ok = typeMap[d.NextBlindingOverride.TlvType()] + if ok && val == nil { + d.NextBlindingOverride = tlv.SomeRecordT(blindingOverride) + } + + if val, ok := typeMap[d.RelayInfo.TlvType()]; ok && val == nil { + d.RelayInfo = tlv.SomeRecordT(relayInfo) + } + + if val, ok := typeMap[d.Constraints.TlvType()]; ok && val == nil { + d.Constraints = tlv.SomeRecordT(constraints) + } + + if val, ok := typeMap[d.Features.TlvType()]; ok && val == nil { + d.Features = tlv.SomeRecordT(features) + } + + return &d, nil +} + +// EncodeBlindedRouteData encodes the blinded route data provided. +func EncodeBlindedRouteData(data *BlindedRouteData) ([]byte, error) { + var ( + e lnwire.ExtraOpaqueData + recordProducers = make([]tlv.RecordProducer, 0, 5) + ) + + data.Padding.WhenSome(func(p tlv.RecordT[tlv.TlvType1, []byte]) { + recordProducers = append(recordProducers, &p) + }) + + data.ShortChannelID.WhenSome(func(scid tlv.RecordT[tlv.TlvType2, + lnwire.ShortChannelID]) { + + recordProducers = append(recordProducers, &scid) + }) + + data.NextNodeID.WhenSome(func(f tlv.RecordT[tlv.TlvType4, + *btcec.PublicKey]) { + + recordProducers = append(recordProducers, &f) + }) + + data.PathID.WhenSome(func(pathID tlv.RecordT[tlv.TlvType6, []byte]) { + recordProducers = append(recordProducers, &pathID) + }) + + data.NextBlindingOverride.WhenSome(func(pk tlv.RecordT[tlv.TlvType8, + *btcec.PublicKey]) { + + recordProducers = append(recordProducers, &pk) + }) + + data.RelayInfo.WhenSome(func(r tlv.RecordT[tlv.TlvType10, + PaymentRelayInfo]) { + + recordProducers = append(recordProducers, &r) + }) + + data.Constraints.WhenSome(func(cs tlv.RecordT[tlv.TlvType12, + PaymentConstraints]) { + + recordProducers = append(recordProducers, &cs) + }) + + data.Features.WhenSome(func(f tlv.RecordT[tlv.TlvType14, + lnwire.FeatureVector]) { + + recordProducers = append(recordProducers, &f) + }) + + if err := e.PackRecords(recordProducers...); err != nil { + return nil, err + } + + return e[:], nil +} + +// PadBy adds "n" padding bytes to the BlindedRouteData using the Padding field. +// Callers should be aware that the total payload size will change by more than +// "n" since the "n" bytes will be prefixed by BigSize type and length fields. +// Callers may need to call PadBy iteratively until each encrypted data packet +// is the same size and so each call will overwrite the Padding record. +// Note that calling PadBy with an n value of 0 will still result in a zero +// length TLV entry being added. +func (b *BlindedRouteData) PadBy(n int) { + b.Padding = tlv.SomeRecordT( + tlv.NewPrimitiveRecord[tlv.TlvType1](make([]byte, n)), + ) +} + +// PaymentRelayInfo describes the relay policy for a blinded path. +type PaymentRelayInfo struct { + // CltvExpiryDelta is the expiry delta for the payment. + CltvExpiryDelta uint16 + + // FeeRate is the fee rate that will be charged per millionth of a + // satoshi. + FeeRate uint32 + + // BaseFee is the per-htlc fee charged in milli-satoshis. + BaseFee lnwire.MilliSatoshi +} + +// Record creates a tlv.Record that encodes the payment relay (type 10) type for +// an encrypted blob payload. +func (i *PaymentRelayInfo) Record() tlv.Record { + return tlv.MakeDynamicRecord( + 10, &i, func() uint64 { + // uint16 + uint32 + tuint32 + return 2 + 4 + tlv.SizeTUint32(uint32(i.BaseFee)) + }, encodePaymentRelay, decodePaymentRelay, + ) +} + +func encodePaymentRelay(w io.Writer, val interface{}, buf *[8]byte) error { + if t, ok := val.(**PaymentRelayInfo); ok { + relayInfo := *t + + // Just write our first 6 bytes directly. + binary.BigEndian.PutUint16(buf[:2], relayInfo.CltvExpiryDelta) + binary.BigEndian.PutUint32(buf[2:6], relayInfo.FeeRate) + if _, err := w.Write(buf[0:6]); err != nil { + return err + } + + baseFee := uint32(relayInfo.BaseFee) + + // We can safely reuse buf here because we overwrite its + // contents. + return tlv.ETUint32(w, &baseFee, buf) + } + + return tlv.NewTypeForEncodingErr(val, "**hop.PaymentRelayInfo") +} + +func decodePaymentRelay(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if t, ok := val.(**PaymentRelayInfo); ok && l <= 10 { + scratch := make([]byte, l) + + n, err := io.ReadFull(r, scratch) + if err != nil { + return err + } + + // We expect at least 6 bytes, because we have 2 bytes for + // cltv delta and 4 bytes for fee rate. + if n < 6 { + return tlv.NewTypeForDecodingErr(val, + "*hop.paymentRelayInfo", uint64(n), 6) + } + + relayInfo := *t + + relayInfo.CltvExpiryDelta = binary.BigEndian.Uint16( + scratch[0:2], + ) + relayInfo.FeeRate = binary.BigEndian.Uint32(scratch[2:6]) + + // To be able to re-use the DTUint32 function we create a + // buffer with just the bytes holding the variable length u32. + // If the base fee is zero, this will be an empty buffer, which + // is okay. + b := bytes.NewBuffer(scratch[6:]) + + var baseFee uint32 + err = tlv.DTUint32(b, &baseFee, buf, l-6) + if err != nil { + return err + } + + relayInfo.BaseFee = lnwire.MilliSatoshi(baseFee) + + return nil + } + + return tlv.NewTypeForDecodingErr(val, "*hop.paymentRelayInfo", l, 10) +} + +// PaymentConstraints is a set of restrictions on a payment. +type PaymentConstraints struct { + // MaxCltvExpiry is the maximum expiry height for the payment. + MaxCltvExpiry uint32 + + // HtlcMinimumMsat is the minimum htlc size for the payment. + HtlcMinimumMsat lnwire.MilliSatoshi +} + +func (p *PaymentConstraints) Record() tlv.Record { + return tlv.MakeDynamicRecord( + 12, &p, func() uint64 { + // uint32 + tuint64. + return 4 + tlv.SizeTUint64(uint64( + p.HtlcMinimumMsat, + )) + }, + encodePaymentConstraints, decodePaymentConstraints, + ) +} + +func encodePaymentConstraints(w io.Writer, val interface{}, + buf *[8]byte) error { + + if c, ok := val.(**PaymentConstraints); ok { + constraints := *c + + binary.BigEndian.PutUint32(buf[:4], constraints.MaxCltvExpiry) + if _, err := w.Write(buf[:4]); err != nil { + return err + } + + // We can safely re-use buf here because we overwrite its + // contents. + htlcMsat := uint64(constraints.HtlcMinimumMsat) + + return tlv.ETUint64(w, &htlcMsat, buf) + } + + return tlv.NewTypeForEncodingErr(val, "**PaymentConstraints") +} + +func decodePaymentConstraints(r io.Reader, val interface{}, buf *[8]byte, + l uint64) error { + + if c, ok := val.(**PaymentConstraints); ok && l <= 12 { + scratch := make([]byte, l) + + n, err := io.ReadFull(r, scratch) + if err != nil { + return err + } + + // We expect at least 4 bytes for our uint32. + if n < 4 { + return tlv.NewTypeForDecodingErr(val, + "*paymentConstraints", uint64(n), 4) + } + + payConstraints := *c + + payConstraints.MaxCltvExpiry = binary.BigEndian.Uint32( + scratch[:4], + ) + + // This could be empty if our minimum is zero, that's okay. + var ( + b = bytes.NewBuffer(scratch[4:]) + minHtlc uint64 + ) + + err = tlv.DTUint64(b, &minHtlc, buf, l-4) + if err != nil { + return err + } + payConstraints.HtlcMinimumMsat = lnwire.MilliSatoshi(minHtlc) + + return nil + } + + return tlv.NewTypeForDecodingErr(val, "**PaymentConstraints", l, l) +} diff --git a/payments/db/migration1/record/custom_records.go b/payments/db/migration1/record/custom_records.go new file mode 100644 index 000000000..01952c24e --- /dev/null +++ b/payments/db/migration1/record/custom_records.go @@ -0,0 +1,31 @@ +package record + +import ( + "fmt" +) + +const ( + // CustomTypeStart is the start of the custom tlv type range as defined + // in BOLT 01. + CustomTypeStart = 65536 +) + +// CustomSet stores a set of custom key/value pairs. +type CustomSet map[uint64][]byte + +// Validate checks that all custom records are in the custom type range. +func (c CustomSet) Validate() error { + for key := range c { + if key < CustomTypeStart { + return fmt.Errorf("no custom records with types "+ + "below %v allowed", CustomTypeStart) + } + } + + return nil +} + +// IsKeysend checks if the custom records contain the key send type. +func (c CustomSet) IsKeysend() bool { + return c[KeySendType] != nil +} diff --git a/payments/db/migration1/record/experimental.go b/payments/db/migration1/record/experimental.go new file mode 100644 index 000000000..3aff0ff26 --- /dev/null +++ b/payments/db/migration1/record/experimental.go @@ -0,0 +1,6 @@ +package record + +const ( + // KeySendType is the custom record identifier for keysend preimages. + KeySendType uint64 = 5482373484 +) diff --git a/payments/db/migration1/record/hop.go b/payments/db/migration1/record/hop.go new file mode 100644 index 000000000..e5c0884f1 --- /dev/null +++ b/payments/db/migration1/record/hop.go @@ -0,0 +1,99 @@ +package record + +import ( + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/tlv" +) + +const ( + // AmtOnionType is the type used in the onion to reference the amount to + // send to the next hop. + AmtOnionType tlv.Type = 2 + + // LockTimeTLV is the type used in the onion to reference the CLTV + // value that should be used for the next hop's HTLC. + LockTimeOnionType tlv.Type = 4 + + // NextHopOnionType is the type used in the onion to reference the ID + // of the next hop. + NextHopOnionType tlv.Type = 6 + + // EncryptedDataOnionType is the type used to include encrypted data + // provided by the receiver in the onion for use in blinded paths. + EncryptedDataOnionType tlv.Type = 10 + + // BlindingPointOnionType is the type used to include receiver provided + // ephemeral keys in the onion that are used in blinded paths. + BlindingPointOnionType tlv.Type = 12 + + // MetadataOnionType is the type used in the onion for the payment + // metadata. + MetadataOnionType tlv.Type = 16 + + // TotalAmtMsatBlindedType is the type used in the onion for the total + // amount field that is included in the final hop for blinded payments. + TotalAmtMsatBlindedType tlv.Type = 18 +) + +// NewAmtToFwdRecord creates a tlv.Record that encodes the amount_to_forward +// (type 2) for an onion payload. +func NewAmtToFwdRecord(amt *uint64) tlv.Record { + return tlv.MakeDynamicRecord( + AmtOnionType, amt, func() uint64 { + return tlv.SizeTUint64(*amt) + }, + tlv.ETUint64, tlv.DTUint64, + ) +} + +// NewLockTimeRecord creates a tlv.Record that encodes the outgoing_cltv_value +// (type 4) for an onion payload. +func NewLockTimeRecord(lockTime *uint32) tlv.Record { + return tlv.MakeDynamicRecord( + LockTimeOnionType, lockTime, func() uint64 { + return tlv.SizeTUint32(*lockTime) + }, + tlv.ETUint32, tlv.DTUint32, + ) +} + +// NewNextHopIDRecord creates a tlv.Record that encodes the short_channel_id +// (type 6) for an onion payload. +func NewNextHopIDRecord(cid *uint64) tlv.Record { + return tlv.MakePrimitiveRecord(NextHopOnionType, cid) +} + +// NewEncryptedDataRecord creates a tlv.Record that encodes the encrypted_data +// (type 10) record for an onion payload. +func NewEncryptedDataRecord(data *[]byte) tlv.Record { + return tlv.MakePrimitiveRecord(EncryptedDataOnionType, data) +} + +// NewBlindingPointRecord creates a tlv.Record that encodes the blinding_point +// (type 12) record for an onion payload. +func NewBlindingPointRecord(point **btcec.PublicKey) tlv.Record { + return tlv.MakePrimitiveRecord(BlindingPointOnionType, point) +} + +// NewMetadataRecord creates a tlv.Record that encodes the metadata (type 10) +// for an onion payload. +func NewMetadataRecord(metadata *[]byte) tlv.Record { + return tlv.MakeDynamicRecord( + MetadataOnionType, metadata, + func() uint64 { + return uint64(len(*metadata)) + }, + tlv.EVarBytes, tlv.DVarBytes, + ) +} + +// NewTotalAmtMsatBlinded creates a tlv.Record that encodes the +// total_amount_msat for the final an onion payload within a blinded route. +func NewTotalAmtMsatBlinded(amt *uint64) tlv.Record { + return tlv.MakeDynamicRecord( + TotalAmtMsatBlindedType, amt, func() uint64 { + return tlv.SizeTUint64(*amt) + }, + tlv.ETUint64, tlv.DTUint64, + ) +} diff --git a/payments/db/migration1/record/mpp.go b/payments/db/migration1/record/mpp.go new file mode 100644 index 000000000..576ada776 --- /dev/null +++ b/payments/db/migration1/record/mpp.go @@ -0,0 +1,112 @@ +package record + +import ( + "fmt" + "io" + + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/tlv" +) + +// MPPOnionType is the type used in the onion to reference the MPP fields: +// total_amt and payment_addr. +const MPPOnionType tlv.Type = 8 + +// MPP is a record that encodes the fields necessary for multi-path payments. +type MPP struct { + // paymentAddr is a random, receiver-generated value used to avoid + // collisions with concurrent payers. + paymentAddr [32]byte + + // totalMsat is the total value of the payment, potentially spread + // across more than one HTLC. + totalMsat lnwire.MilliSatoshi +} + +// NewMPP generates a new MPP record with the given total and payment address. +func NewMPP(total lnwire.MilliSatoshi, addr [32]byte) *MPP { + return &MPP{ + paymentAddr: addr, + totalMsat: total, + } +} + +// PaymentAddr returns the payment address contained in the MPP record. +func (r *MPP) PaymentAddr() [32]byte { + return r.paymentAddr +} + +// TotalMsat returns the total value of an MPP payment in msats. +func (r *MPP) TotalMsat() lnwire.MilliSatoshi { + return r.totalMsat +} + +// MPPEncoder writes the MPP record to the provided io.Writer. +func MPPEncoder(w io.Writer, val interface{}, buf *[8]byte) error { + if v, ok := val.(*MPP); ok { + err := tlv.EBytes32(w, &v.paymentAddr, buf) + if err != nil { + return err + } + + return tlv.ETUint64T(w, uint64(v.totalMsat), buf) + } + return tlv.NewTypeForEncodingErr(val, "MPP") +} + +const ( + // minMPPLength is the minimum length of a serialized MPP TLV record, + // which occurs when the truncated encoding of total_amt_msat takes 0 + // bytes, leaving only the payment_addr. + minMPPLength = 32 + + // maxMPPLength is the maximum length of a serialized MPP TLV record, + // which occurs when the truncated encoding of total_amt_msat takes 8 + // bytes. + maxMPPLength = 40 +) + +// MPPDecoder reads the MPP record to the provided io.Reader. +func MPPDecoder(r io.Reader, val interface{}, buf *[8]byte, l uint64) error { + if v, ok := val.(*MPP); ok && minMPPLength <= l && l <= maxMPPLength { + if err := tlv.DBytes32(r, &v.paymentAddr, buf, 32); err != nil { + return err + } + + var total uint64 + if err := tlv.DTUint64(r, &total, buf, l-32); err != nil { + return err + } + v.totalMsat = lnwire.MilliSatoshi(total) + + return nil + } + return tlv.NewTypeForDecodingErr(val, "MPP", l, maxMPPLength) +} + +// Record returns a tlv.Record that can be used to encode or decode this record. +func (r *MPP) Record() tlv.Record { + // Fixed-size, 32 byte payment address followed by truncated 64-bit + // total msat. + size := func() uint64 { + return 32 + tlv.SizeTUint64(uint64(r.totalMsat)) + } + + return tlv.MakeDynamicRecord( + MPPOnionType, r, size, MPPEncoder, MPPDecoder, + ) +} + +// PayloadSize returns the size this record takes up in encoded form. +func (r *MPP) PayloadSize() uint64 { + return 32 + tlv.SizeTUint64(uint64(r.totalMsat)) +} + +// String returns a human-readable representation of the mpp payload field. +func (r *MPP) String() string { + if r == nil { + return "" + } + + return fmt.Sprintf("total=%v, addr=%x", r.totalMsat, r.paymentAddr) +} diff --git a/payments/db/migration1/route.go b/payments/db/migration1/route.go new file mode 100644 index 000000000..bdf8c6ff0 --- /dev/null +++ b/payments/db/migration1/route.go @@ -0,0 +1,138 @@ +package migration1 + +import ( + "fmt" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/record" + "github.com/lightningnetwork/lnd/tlv" +) + +// Vertex is a frozen local type representing a 33-byte compressed public key, +// equivalent to route.Vertex at the time this migration was written. +type Vertex [33]byte + +// NewVertex returns a new Vertex from a compressed public key. +func NewVertex(pub *btcec.PublicKey) Vertex { + var v Vertex + copy(v[:], pub.SerializeCompressed()) + + return v +} + +// NewVertexFromBytes returns a new Vertex from a serialized compressed public +// key byte slice. +func NewVertexFromBytes(b []byte) (Vertex, error) { + if len(b) != 33 { + return Vertex{}, fmt.Errorf("invalid vertex length %d, "+ + "expected 33", len(b)) + } + + var v Vertex + copy(v[:], b) + + return v, nil +} + +// Hop is a frozen local snapshot of route.Hop, containing exactly the fields +// that existed at the time this migration was written. This ensures the +// migration's serialization boundary is independent of future changes to +// route.Hop, in particular the planned removal of LegacyPayload once the KV +// backend is phased out. +type Hop struct { + // PubKeyBytes is the raw bytes of the public key of the target node. + PubKeyBytes Vertex + + // ChannelID is the unique channel ID for the channel. + ChannelID uint64 + + // OutgoingTimeLock is the timelock value that should be used when + // crafting the _outgoing_ HTLC from this hop. + OutgoingTimeLock uint32 + + // AmtToForward is the amount that this hop will forward to the next + // hop. + AmtToForward lnwire.MilliSatoshi + + // MPP encapsulates the data required for option_mpp. This field should + // only be set for the final hop. + MPP *record.MPP + + // AMP encapsulates the data required for option_amp. This field should + // only be set for the final hop. + AMP *record.AMP + + // CustomRecords if non-nil are a set of additional TLV records that + // should be included in the forwarding instructions for this node. + CustomRecords record.CustomSet + + // LegacyPayload signals that this node doesn't understand the new TLV + // payload, so the legacy payload format must be used. + // + // NOTE: This field is preserved here even though it is marked for + // removal in the live route.Hop, because old KV data may have been + // serialized with LegacyPayload=true and the migration must be able to + // deserialize it correctly. + LegacyPayload bool + + // Metadata is additional data sent along with the payment to the + // payee. + Metadata []byte + + // EncryptedData is an encrypted data blob included for hops that are + // part of a blinded route. + EncryptedData []byte + + // BlindingPoint is an ephemeral public key used by introduction nodes + // in blinded routes. + BlindingPoint *btcec.PublicKey + + // TotalAmtMsat is the total amount for a blinded payment. This field + // should only be set for the final hop in a blinded path. + TotalAmtMsat lnwire.MilliSatoshi +} + +// Route is a frozen local snapshot of route.Route, containing exactly the +// fields that existed at the time this migration was written. +type Route struct { + // TotalTimeLock is the cumulative (final) time lock across the entire + // route. + TotalTimeLock uint32 + + // TotalAmount is the total amount of funds required to complete a + // payment over this route, including fees. + TotalAmount lnwire.MilliSatoshi + + // SourcePubKey is the pubkey of the node where this route originates. + SourcePubKey Vertex + + // Hops contains details concerning the specific forwarding details at + // each hop. + Hops []*Hop + + // FirstHopAmount is the amount that should actually be sent to the + // first hop. Only differs from TotalAmount for custom channels. + FirstHopAmount tlv.RecordT[ + tlv.TlvType0, tlv.BigSizeT[lnwire.MilliSatoshi], + ] + + // FirstHopWireCustomRecords is a set of custom TLV records to include + // in the wire message sent to the first hop. + FirstHopWireCustomRecords lnwire.CustomRecords +} + +// FinalHop returns the last hop in the route. +func (r *Route) FinalHop() *Hop { + return r.Hops[len(r.Hops)-1] +} + +// ReceiverAmt returns the amount forwarded to the final hop. +func (r *Route) ReceiverAmt() lnwire.MilliSatoshi { + return r.FinalHop().AmtToForward +} + +// TotalFees returns the total fees paid along the route. +func (r *Route) TotalFees() lnwire.MilliSatoshi { + return r.TotalAmount - r.ReceiverAmt() +} diff --git a/payments/db/migration1/sql_converters.go b/payments/db/migration1/sql_converters.go new file mode 100644 index 000000000..50d129bf7 --- /dev/null +++ b/payments/db/migration1/sql_converters.go @@ -0,0 +1,274 @@ +package migration1 + +import ( + "bytes" + "fmt" + "strconv" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/record" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/tlv" +) + +// dbPaymentToCreationInfo converts database payment data to the +// PaymentCreationInfo struct. +func dbPaymentToCreationInfo(paymentIdentifier []byte, amountMsat int64, + createdAt time.Time, intentPayload []byte, + firstHopCustomRecords lnwire.CustomRecords) *PaymentCreationInfo { + + // This is the payment hash for non-AMP payments and the SetID for AMP + // payments. + var identifier lntypes.Hash + copy(identifier[:], paymentIdentifier) + + return &PaymentCreationInfo{ + PaymentIdentifier: identifier, + Value: lnwire.MilliSatoshi(amountMsat), + // The creation time is stored in the database as UTC but here + // we convert it to local time. + CreationTime: createdAt.Local(), + PaymentRequest: intentPayload, + FirstHopCustomRecords: firstHopCustomRecords, + } +} + +// dbAttemptToHTLCAttempt converts a database HTLC attempt to an HTLCAttempt. +func dbAttemptToHTLCAttempt(dbAttempt sqlc.FetchHtlcAttemptsForPaymentsRow, + hops []sqlc.FetchHopsForAttemptsRow, + hopCustomRecords map[int64][]sqlc.PaymentHopCustomRecord, + routeCustomRecords []sqlc.PaymentAttemptFirstHopCustomRecord) ( + *HTLCAttempt, error) { + + // Convert route-level first hop custom records to CustomRecords map. + var firstHopWireCustomRecords lnwire.CustomRecords + if len(routeCustomRecords) > 0 { + firstHopWireCustomRecords = make(lnwire.CustomRecords) + for _, record := range routeCustomRecords { + firstHopWireCustomRecords[uint64(record.Key)] = + record.Value + } + } + + // Build the route from the database data. + route, err := dbDataToRoute( + hops, hopCustomRecords, dbAttempt.FirstHopAmountMsat, + dbAttempt.RouteTotalTimeLock, dbAttempt.RouteTotalAmount, + dbAttempt.RouteSourceKey, firstHopWireCustomRecords, + ) + if err != nil { + return nil, fmt.Errorf("failed to convert to route: %w", + err) + } + + hash, err := lntypes.MakeHash(dbAttempt.PaymentHash) + if err != nil { + return nil, fmt.Errorf("failed to parse payment "+ + "hash: %w", err) + } + + // Create the attempt info. + var sessionKey [32]byte + copy(sessionKey[:], dbAttempt.SessionKey) + + info := HTLCAttemptInfo{ + AttemptID: uint64(dbAttempt.AttemptIndex), + sessionKey: sessionKey, + Route: *route, + AttemptTime: dbAttempt.AttemptTime, + Hash: &hash, + } + + attempt := &HTLCAttempt{ + HTLCAttemptInfo: info, + } + + // If there's no resolution type, the attempt is still in-flight. + // Return early without processing settlement or failure info. + if !dbAttempt.ResolutionType.Valid { + return attempt, nil + } + + // Add settlement info if present. + if HTLCAttemptResolutionType(dbAttempt.ResolutionType.Int32) == + HTLCAttemptResolutionSettled { + + var preimage lntypes.Preimage + copy(preimage[:], dbAttempt.SettlePreimage) + + attempt.Settle = &HTLCSettleInfo{ + Preimage: preimage, + SettleTime: dbAttempt.ResolutionTime.Time, + } + } + + // Add failure info if present. + if HTLCAttemptResolutionType(dbAttempt.ResolutionType.Int32) == + HTLCAttemptResolutionFailed { + + failure := &HTLCFailInfo{ + FailTime: dbAttempt.ResolutionTime.Time, + } + + if dbAttempt.HtlcFailReason.Valid { + failure.Reason = HTLCFailReason( + dbAttempt.HtlcFailReason.Int32, + ) + } + + if dbAttempt.FailureSourceIndex.Valid { + failure.FailureSourceIndex = uint32( + dbAttempt.FailureSourceIndex.Int32, + ) + } + + // Decode the failure message if present. + if len(dbAttempt.FailureMsg) > 0 { + msg, err := lnwire.DecodeFailureMessage( + bytes.NewReader(dbAttempt.FailureMsg), 0, + ) + if err != nil { + return nil, fmt.Errorf("failed to decode "+ + "failure message: %w", err) + } + failure.Message = msg + } + + attempt.Failure = failure + } + + return attempt, nil +} + +// dbDataToRoute converts database route data to a Route. +func dbDataToRoute(hops []sqlc.FetchHopsForAttemptsRow, + hopCustomRecords map[int64][]sqlc.PaymentHopCustomRecord, + firstHopAmountMsat int64, totalTimeLock int32, totalAmount int64, + sourceKey []byte, firstHopWireCustomRecords lnwire.CustomRecords) ( + *Route, error) { + + if len(hops) == 0 { + return nil, fmt.Errorf("no hops provided") + } + + // Hops are already sorted by hop_index from the SQL query. + routeHops := make([]*Hop, len(hops)) + + for i, hop := range hops { + pubKey, err := NewVertexFromBytes(hop.PubKey) + if err != nil { + return nil, fmt.Errorf("failed to parse pub key: %w", + err) + } + + var channelID uint64 + if hop.Scid != "" { + // The SCID is stored as a string representation + // of the uint64. + var err error + channelID, err = strconv.ParseUint(hop.Scid, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse "+ + "scid: %w", err) + } + } + + routeHop := &Hop{ + PubKeyBytes: pubKey, + ChannelID: channelID, + OutgoingTimeLock: uint32(hop.OutgoingTimeLock), + AmtToForward: lnwire.MilliSatoshi(hop.AmtToForward), + } + + // Add MPP record if present. + if len(hop.MppPaymentAddr) > 0 { + var paymentAddr [32]byte + copy(paymentAddr[:], hop.MppPaymentAddr) + routeHop.MPP = record.NewMPP( + lnwire.MilliSatoshi(hop.MppTotalMsat.Int64), + paymentAddr, + ) + } + + // Add AMP record if present. + if len(hop.AmpRootShare) > 0 { + var rootShare [32]byte + copy(rootShare[:], hop.AmpRootShare) + var setID [32]byte + copy(setID[:], hop.AmpSetID) + + routeHop.AMP = record.NewAMP( + rootShare, setID, + uint32(hop.AmpChildIndex.Int32), + ) + } + + // Add blinding point if present (only for introduction node + // in blinded route). + if len(hop.BlindingPoint) > 0 { + pubKey, err := btcec.ParsePubKey(hop.BlindingPoint) + if err != nil { + return nil, fmt.Errorf("failed to parse "+ + "blinding point: %w", err) + } + routeHop.BlindingPoint = pubKey + } + + // Add encrypted data if present (for all blinded hops). + if len(hop.EncryptedData) > 0 { + routeHop.EncryptedData = hop.EncryptedData + } + + // Add total amount if present (only for final hop in blinded + // route). + if hop.BlindedPathTotalAmt.Valid { + routeHop.TotalAmtMsat = lnwire.MilliSatoshi( + hop.BlindedPathTotalAmt.Int64, + ) + } + + // Add hop-level custom records. + if records, ok := hopCustomRecords[hop.ID]; ok { + routeHop.CustomRecords = make( + record.CustomSet, + ) + for _, rec := range records { + routeHop.CustomRecords[uint64(rec.Key)] = + rec.Value + } + } + + // Add metadata if present. + if len(hop.MetaData) > 0 { + routeHop.Metadata = hop.MetaData + } + + routeHops[i] = routeHop + } + + // Parse the source node public key. + var sourceNode Vertex + copy(sourceNode[:], sourceKey) + + route := &Route{ + TotalTimeLock: uint32(totalTimeLock), + TotalAmount: lnwire.MilliSatoshi(totalAmount), + SourcePubKey: sourceNode, + Hops: routeHops, + FirstHopWireCustomRecords: firstHopWireCustomRecords, + } + + // Set the first hop amount if it is set. + if firstHopAmountMsat != 0 { + route.FirstHopAmount = tlv.NewRecordT[tlv.TlvType0]( + tlv.NewBigSizeT(lnwire.MilliSatoshi( + firstHopAmountMsat, + )), + ) + } + + return route, nil +} diff --git a/payments/db/migration1/sql_migration.go b/payments/db/migration1/sql_migration.go new file mode 100644 index 000000000..2bda95f57 --- /dev/null +++ b/payments/db/migration1/sql_migration.go @@ -0,0 +1,820 @@ +package migration1 + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "strconv" + "time" + + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "golang.org/x/time/rate" +) + +// MigrationStats tracks migration progress. +type MigrationStats struct { + TotalPayments int64 + SuccessfulPayments int64 + FailedPayments int64 + InFlightPayments int64 + InitiatedPayments int64 + TotalAttempts int64 + SettledAttempts int64 + FailedAttempts int64 + InFlightAttempts int64 + TotalHops int64 + DuplicatePayments int64 + DuplicateEntries int64 + SkippedPayments int64 + MigrationDuration time.Duration +} + +// MigratePaymentsKVToSQL migrates payments from KV to SQL and validates +// migrated data in batches. Callers are responsible for executing this within +// a single SQL transaction if atomicity is required. +func MigratePaymentsKVToSQL(ctx context.Context, kvBackend kvdb.Backend, + sqlDB SQLMigrationQueries, cfg *SQLStoreConfig) error { + + if cfg == nil { + return fmt.Errorf("missing SQL store config for migration") + } + + if cfg.QueryCfg == nil { + return fmt.Errorf("missing SQL store config for " + + "validation") + } + + if cfg.QueryCfg.MaxBatchSize == 0 { + return fmt.Errorf("invalid max batch size for " + + "validation") + } + + stats := &MigrationStats{} + startTime := time.Now() + + log.Infof("Starting payment migration from KV to SQL...") + + var ( + validationBatch []migratedPaymentRef + validatedPayments int64 + + reportInterval = rate.Sometimes{Interval: 5 * time.Second} + validationInterval = rate.Sometimes{Interval: 5 * time.Second} + ) + + // Open the KV backend in read-only mode. + err := kvBackend.View(func(kvTx kvdb.RTx) error { + // In case we start with an empty database, there are no + // payments to migrate. + paymentsBucket := kvTx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + log.Infof("No payments bucket found - database is " + + "empty") + + return nil + } + + // The index bucket maps sequence number -> payment hash. + indexes := kvTx.ReadBucket(paymentsIndexBucket) + if indexes == nil { + return fmt.Errorf("index bucket does not exist") + } + + // We iterate over all sequence numbers in the index bucket to + // make sure we have the correct order of payments. Otherwise, + // if we just loop over the payments bucket, we might get the + // payments not in the chronological order but rather the + // lexicographical order of the payment hashes. + return indexes.ForEach(func(seqKey, indexVal []byte) error { + // Progress reporting based on time + actual work done. + reportProgress := func() { + elapsed := time.Since(startTime) + if elapsed <= 0 { + return + } + + if stats.TotalPayments == 0 { + return + } + + paymentRate := float64(stats.TotalPayments) / + elapsed.Seconds() + attemptRate := float64(stats.TotalAttempts) / + elapsed.Seconds() + + log.Infof("Progress: %d payments, %d "+ + "attempts, %d hops | Rate: %.1f "+ + "pmt/s, %.1f att/s | Elapsed: %v", + stats.TotalPayments, + stats.TotalAttempts, stats.TotalHops, + paymentRate, attemptRate, + elapsed.Round(time.Second), + ) + } + + reportInterval.Do(reportProgress) + + r := bytes.NewReader(indexVal) + paymentHash, err := deserializePaymentIndex(r) + if err != nil { + return err + } + + paymentBucket := paymentsBucket.NestedReadBucket( + paymentHash[:], + ) + if paymentBucket == nil { + // We skip the entry in case this sequence + // number does not have a corresponding + // payment bucket. But aborting would not help + // either because it is just a db inconsistency. + log.Warnf("Missing bucket for payment %x", + paymentHash[:8]) + + stats.SkippedPayments++ + + return nil + } + + // Every payment bucket should have a sequence number + // which is also important to check for duplicates. + seqBytes := paymentBucket.Get(paymentSequenceKey) + if seqBytes == nil { + return ErrNoSequenceNumber + } + + // Skip duplicates. They are migrated into the + // payment_duplicates table when the primary payment is + // processed. + if !bytes.Equal(seqBytes, seqKey) { + return nil + } + + // Fetch the payment from the kv store. + payment, err := fetchPayment(paymentBucket) + if err != nil { + return fmt.Errorf("fetch payment %x: %w", + paymentHash[:8], err) + } + + // Migrate the payment to the SQL database. + paymentID, err := migratePayment( + ctx, payment, paymentHash, sqlDB, stats, + ) + if err != nil { + return fmt.Errorf("migrate payment %x: %w", + paymentHash[:8], err) + } + + // Check for duplicates. + dupBucket := paymentBucket.NestedReadBucket( + duplicatePaymentsBucket, + ) + if dupBucket != nil { + err = migrateDuplicatePayments( + ctx, dupBucket, paymentHash, + paymentID, sqlDB, stats, + ) + if err != nil { + return fmt.Errorf("migrate duplicates "+ + "%x: %w", paymentHash[:8], + err) + } + } + + // Add the payment to the validation batch. + validationBatch = append( + validationBatch, migratedPaymentRef{ + Hash: paymentHash, + PaymentID: paymentID, + }, + ) + if uint32(len(validationBatch)) >= + cfg.QueryCfg.MaxBatchSize { + + err := validateMigratedPaymentBatch( + ctx, kvBackend, sqlDB, + cfg, + validationBatch, + ) + if err != nil { + return err + } + + validatedPayments += int64( + len(validationBatch), + ) + + // Log validation progress periodically. + validationInterval.Do(func() { + log.Infof("Validated %d/%d "+ + "payments", + validatedPayments, + stats.TotalPayments, + ) + }) + + validationBatch = validationBatch[:0] + } + + return nil + }) + }, func() {}) + + if err != nil { + return fmt.Errorf("migrate payments: %w", err) + } + + // Validate any remaining payments in the batch. + if len(validationBatch) > 0 { + if err := validateMigratedPaymentBatch( + ctx, kvBackend, sqlDB, cfg, validationBatch, + ); err != nil { + return err + } + + validatedPayments += int64(len(validationBatch)) + log.Infof("Validated %d/%d payments", validatedPayments, + stats.TotalPayments) + } + + // Validate the total number of payments as an additional sanity check. + if err := validatePaymentCounts( + ctx, sqlDB, stats.TotalPayments, + ); err != nil { + return err + } + + stats.MigrationDuration = time.Since(startTime) + + printMigrationSummary(stats) + + return nil +} + +// normalizeTimeForSQL converts a timestamp into the representation we persist +// and compare against in SQL: +// - drops any monotonic clock reading (SQL can't store it), +// - forces UTC for deterministic comparisons across environments. +// +// A zero time is returned unchanged. +func normalizeTimeForSQL(t time.Time) time.Time { + if t.IsZero() { + return t + } + + return time.Unix(0, t.UnixNano()).UTC() +} + +// migratePayment migrates a single payment from KV to SQL. +func migratePayment(ctx context.Context, payment *MPPayment, hash lntypes.Hash, + sqlDB SQLMigrationQueries, stats *MigrationStats) (int64, error) { + + // Update migration stats based on payment status. + switch payment.Status { + case StatusSucceeded: + stats.SuccessfulPayments++ + + case StatusFailed: + stats.FailedPayments++ + + case StatusInFlight: + stats.InFlightPayments++ + + case StatusInitiated: + stats.InitiatedPayments++ + } + + // Prepare fail reason for SQL insert. + var failReason sql.NullInt32 + if payment.FailureReason != nil { + failReason = sql.NullInt32{ + Int32: int32(*payment.FailureReason), + Valid: true, + } + } + + // Insert payment using migration query. + paymentID, err := sqlDB.InsertPaymentMig( + ctx, sqlc.InsertPaymentMigParams{ + AmountMsat: int64(payment.Info.Value), + CreatedAt: normalizeTimeForSQL( + payment.Info.CreationTime, + ), + PaymentIdentifier: hash[:], + FailReason: failReason, + }) + if err != nil { + return 0, fmt.Errorf("insert payment: %w", err) + } + + // Insert payment intent. + // + // Only insert a row if we have an actual intent payload. For legacy + // hash-only/keysend-style payments, the intent may be absent. + if len(payment.Info.PaymentRequest) > 0 { + _, err = sqlDB.InsertPaymentIntent( + ctx, sqlc.InsertPaymentIntentParams{ + PaymentID: paymentID, + IntentType: int16(PaymentIntentTypeBolt11), + IntentPayload: payment.Info.PaymentRequest, + }, + ) + if err != nil { + return 0, fmt.Errorf("insert intent: %w", err) + } + } + + // Insert first hop custom records (payment level). + for key, value := range payment.Info.FirstHopCustomRecords { + err = sqlDB.InsertPaymentFirstHopCustomRecord(ctx, + sqlc.InsertPaymentFirstHopCustomRecordParams{ + PaymentID: paymentID, + Key: int64(key), + Value: value, + }, + ) + if err != nil { + return 0, fmt.Errorf("insert custom record: %w", err) + } + } + + // Migrate HTLC attempts. + for _, htlc := range payment.HTLCs { + err = migrateHTLCAttempt( + ctx, paymentID, hash, &htlc, sqlDB, stats, + ) + if err != nil { + return 0, fmt.Errorf("migrate attempt %d: %w", + htlc.AttemptID, err) + } + } + + stats.TotalPayments++ + + return paymentID, nil +} + +// migrateHTLCAttempt migrates a single HTLC attempt. +func migrateHTLCAttempt(ctx context.Context, paymentID int64, + parentPaymentHash lntypes.Hash, htlc *HTLCAttempt, sqlDB SQLQueries, + stats *MigrationStats) error { + + // Determine the payment hash for this HTLC attempt. + // + // For AMP payments, each HTLC has its own unique hash. For non-AMP + // payments (MPP, Legacy), all HTLCs use the same hash as the parent + // payment. Older payment attempts may not have the hash stored + // explicitly, in which case we fall back to the parent payment hash + // which is ok since non-AMP payments have a single hash for all HTLCs. + var paymentHash []byte + switch { + case htlc.Hash != nil: + paymentHash = (*htlc.Hash)[:] + + default: + // For older payments where Hash is nil, use the parent payment + // hash. This is consistent with how the router handles these + // legacy payments. + paymentHash = parentPaymentHash[:] + } + + firstHopAmountMsat := int64(htlc.Route.FirstHopAmount.Val.Int()) + + // Get the session key bytes. + sessionKeyBytes := htlc.SessionKey().Serialize() + + // Insert HTLC attempt. + _, err := sqlDB.InsertHtlcAttempt(ctx, sqlc.InsertHtlcAttemptParams{ + PaymentID: paymentID, + AttemptIndex: int64(htlc.AttemptID), + SessionKey: sessionKeyBytes, + AttemptTime: normalizeTimeForSQL(htlc.AttemptTime), + PaymentHash: paymentHash, + FirstHopAmountMsat: firstHopAmountMsat, + RouteTotalTimeLock: int32(htlc.Route.TotalTimeLock), + RouteTotalAmount: int64(htlc.Route.TotalAmount), + RouteSourceKey: htlc.Route.SourcePubKey[:], + }) + if err != nil { + return fmt.Errorf("insert HTLC attempt: %w", err) + } + + // Insert the route-level first hop custom records. + for key, value := range htlc.Route.FirstHopWireCustomRecords { + err = sqlDB.InsertPaymentAttemptFirstHopCustomRecord( + ctx, + sqlc.InsertPaymentAttemptFirstHopCustomRecordParams{ + HtlcAttemptIndex: int64(htlc.AttemptID), + Key: int64(key), + Value: value, + }, + ) + if err != nil { + return fmt.Errorf("insert attempt first hop custom "+ + "record: %w", err) + } + } + + // Insert route hops. + for hopIndex := range htlc.Route.Hops { + hop := htlc.Route.Hops[hopIndex] + err = migrateRouteHop( + ctx, int64(htlc.AttemptID), hopIndex, hop, + sqlDB, stats, + ) + if err != nil { + return fmt.Errorf("migrate hop %d: %w", hopIndex, err) + } + } + + // Handle attempt resolution (settle or fail). + switch { + case htlc.Settle != nil: + // Settled + err = sqlDB.SettleAttempt(ctx, sqlc.SettleAttemptParams{ + AttemptIndex: int64(htlc.AttemptID), + ResolutionTime: normalizeTimeForSQL( + htlc.Settle.SettleTime, + ), + ResolutionType: int32(HTLCAttemptResolutionSettled), + SettlePreimage: htlc.Settle.Preimage[:], + }) + if err != nil { + return fmt.Errorf("settle attempt: %w", err) + } + + stats.SettledAttempts++ + + case htlc.Failure != nil: + var failureMsg bytes.Buffer + if htlc.Failure.Message != nil { + err := lnwire.EncodeFailureMessage( + &failureMsg, htlc.Failure.Message, 0, + ) + if err != nil { + return fmt.Errorf("failed to encode "+ + "failure message: %w", err) + } + } + + err = sqlDB.FailAttempt(ctx, sqlc.FailAttemptParams{ + AttemptIndex: int64(htlc.AttemptID), + ResolutionTime: normalizeTimeForSQL( + htlc.Failure.FailTime, + ), + ResolutionType: int32(HTLCAttemptResolutionFailed), + FailureSourceIndex: sql.NullInt32{ + Int32: int32(htlc.Failure.FailureSourceIndex), + Valid: true, + }, + HtlcFailReason: sql.NullInt32{ + Int32: int32(htlc.Failure.Reason), + Valid: true, + }, + FailureMsg: failureMsg.Bytes(), + }) + if err != nil { + return fmt.Errorf("fail attempt: %w", err) + } + + stats.FailedAttempts++ + + default: + // If the attempt is not settled or failed, it is in flight. + stats.InFlightAttempts++ + } + + stats.TotalAttempts++ + + return nil +} + +// migrateRouteHop migrates a single route hop. +func migrateRouteHop(ctx context.Context, + attemptID int64, hopIndex int, hop *Hop, sqlDB SQLQueries, + stats *MigrationStats) error { + + // Convert channel ID to string representation of uint64. + // The SCID is stored as a decimal string to match the converter + // expectations (sql_converters.go:173). + scidStr := strconv.FormatUint(hop.ChannelID, 10) + + // Insert route hop. + hopID, err := sqlDB.InsertRouteHop(ctx, sqlc.InsertRouteHopParams{ + HtlcAttemptIndex: attemptID, + HopIndex: int32(hopIndex), + PubKey: hop.PubKeyBytes[:], + Scid: scidStr, + OutgoingTimeLock: int32(hop.OutgoingTimeLock), + AmtToForward: int64(hop.AmtToForward), + MetaData: hop.Metadata, + }) + if err != nil { + return fmt.Errorf("insert hop: %w", err) + } + + // Check for blinded route data (route blinding). + if len(hop.EncryptedData) > 0 || hop.BlindingPoint != nil || + hop.TotalAmtMsat != 0 { + + var blindingPoint []byte + if hop.BlindingPoint != nil { + blindingPoint = hop.BlindingPoint.SerializeCompressed() + } + + var totalAmt sql.NullInt64 + if hop.TotalAmtMsat != 0 { + totalAmt = sql.NullInt64{ + Int64: int64(hop.TotalAmtMsat), + Valid: true, + } + } + + err := sqlDB.InsertRouteHopBlinded( + ctx, sqlc.InsertRouteHopBlindedParams{ + HopID: hopID, + EncryptedData: hop.EncryptedData, + BlindingPoint: blindingPoint, + BlindedPathTotalAmt: totalAmt, + }, + ) + if err != nil { + return fmt.Errorf("insert blinded hop: %w", err) + } + } + + // Check for MPP record. + if hop.MPP != nil { + paymentAddr := hop.MPP.PaymentAddr() + err = sqlDB.InsertRouteHopMpp(ctx, sqlc.InsertRouteHopMppParams{ + HopID: hopID, + PaymentAddr: paymentAddr[:], + TotalMsat: int64(hop.MPP.TotalMsat()), + }) + if err != nil { + return fmt.Errorf("insert MPP: %w", err) + } + } + + // Check for AMP record. + if hop.AMP != nil { + rootShare := hop.AMP.RootShare() + setID := hop.AMP.SetID() + err = sqlDB.InsertRouteHopAmp(ctx, sqlc.InsertRouteHopAmpParams{ + HopID: hopID, + RootShare: rootShare[:], + SetID: setID[:], + ChildIndex: int32(hop.AMP.ChildIndex()), + }) + if err != nil { + return fmt.Errorf("insert AMP: %w", err) + } + } + + // Check for custom records. + if hop.CustomRecords != nil { + for tlvType, value := range hop.CustomRecords { + err = sqlDB.InsertPaymentHopCustomRecord( + ctx, + sqlc.InsertPaymentHopCustomRecordParams{ + HopID: hopID, + Key: int64(tlvType), + Value: value, + }, + ) + if err != nil { + return fmt.Errorf("insert hop custom "+ + "record: %w", err) + } + } + } + + stats.TotalHops++ + + return nil +} + +// migrateDuplicatePayments migrates duplicate payments into the dedicated +// payment_duplicates table. +func migrateDuplicatePayments(ctx context.Context, dupBucket kvdb.RBucket, + hash [32]byte, primaryPaymentID int64, sqlDB SQLMigrationQueries, + stats *MigrationStats) error { + + duplicateCount := 0 + + err := dupBucket.ForEach(func(seqBytes, _ []byte) error { + // The duplicates bucket should only contain nested buckets + // keyed by 8-byte sequence numbers. Skip any unexpected keys + // (defensive check for corrupted or malformed data). + if len(seqBytes) != 8 { + log.Warnf("Skipping unexpected key in duplicates "+ + "bucket for payment %x: key length %d, "+ + "expected 8", + hash[:8], len(seqBytes), + ) + + return nil + } + + seqNum := byteOrder.Uint64(seqBytes) + subBucket := dupBucket.NestedReadBucket(seqBytes) + if subBucket == nil { + return nil + } + + duplicateCount++ + log.Infof("Migrating duplicate payment seq=%d for "+ + "payment %x", seqNum, hash[:8]) + + err := migrateSingleDuplicatePayment( + ctx, subBucket, hash, primaryPaymentID, seqNum, + sqlDB, + ) + if err != nil { + return fmt.Errorf( + "migrate duplicate payment seq=%d: %w", + seqNum, err, + ) + } + + return nil + }) + + if duplicateCount > 0 { + stats.DuplicatePayments++ + stats.DuplicateEntries += int64(duplicateCount) + + log.Infof("Payment %x had %d duplicate(s) migrated", hash[:8], + duplicateCount) + } + + return err +} + +// migrateSingleDuplicatePayment inserts a duplicate payment record for the +// given payment hash into payment_duplicates. +func migrateSingleDuplicatePayment(ctx context.Context, dupBucket kvdb.RBucket, + hash [32]byte, primaryPaymentID int64, duplicateSeq uint64, + sqlDB SQLMigrationQueries) error { + + creationData := dupBucket.Get(duplicatePaymentCreationInfoKey) + if creationData == nil { + return fmt.Errorf("duplicate payment seq=%d missing "+ + "creation info (payment=%x)", duplicateSeq, hash[:8]) + } + + creationInfo, err := deserializeDuplicatePaymentCreationInfo( + bytes.NewReader(creationData), + ) + if err != nil { + return fmt.Errorf("deserialize duplicate creation "+ + "info: %w", err) + } + + settleData := dupBucket.Get(duplicatePaymentSettleInfoKey) + failReasonData := dupBucket.Get(duplicatePaymentFailInfoKey) + attemptData := dupBucket.Get(duplicatePaymentAttemptInfoKey) + + if settleData != nil && len(failReasonData) > 0 { + return fmt.Errorf("duplicate payment seq=%d has both "+ + "settle and fail info (payment=%x)", duplicateSeq, + hash[:8]) + } + + var ( + failReason sql.NullInt32 + settlePreimage []byte + settleTime sql.NullTime + ) + + switch { + case settleData != nil: + settlePreimage, settleTime, err = parseDuplicateSettleData( + settleData, + ) + if err != nil { + return err + } + + case len(failReasonData) > 0: + failReason = sql.NullInt32{ + Int32: int32(failReasonData[0]), + Valid: true, + } + + default: + // If the duplicate payment has no settle or fail info, + // we mark it as failed during the migration. Duplicate + // payments were a bug in older versions of LND, so we can be + // sure if a duplicate payment has no failure reason or + // settlement data, the corresponding HTLC for this payment + // has been failed (resolved). + if attemptData == nil { + log.Warnf("Duplicate payment seq=%d has no "+ + "attempt info and no resolution (payment=%x); "+ + "marking failed", duplicateSeq, hash[:8]) + } else { + log.Warnf("Duplicate payment seq=%d has attempt "+ + "info but no resolution (payment=%x); "+ + "marking failed", duplicateSeq, hash[:8]) + } + + failReason = sql.NullInt32{ + Int32: int32(FailureReasonError), + Valid: true, + } + } + + _, err = sqlDB.InsertPaymentDuplicateMig( + ctx, sqlc.InsertPaymentDuplicateMigParams{ + PaymentID: primaryPaymentID, + AmountMsat: int64(creationInfo.Value), + CreatedAt: normalizeTimeForSQL( + creationInfo.CreationTime, + ), + FailReason: failReason, + SettlePreimage: settlePreimage, + SettleTime: settleTime, + }, + ) + if err != nil { + return fmt.Errorf("insert duplicate payment: %w", err) + } + + return nil +} + +// parseDuplicateSettleData extracts settle data from either legacy or modern +// duplicate formats. +func parseDuplicateSettleData(settleData []byte) ([]byte, sql.NullTime, error) { + if len(settleData) == lntypes.PreimageSize { + return append([]byte(nil), settleData...), sql.NullTime{}, nil + } + + settleInfo, err := deserializeHTLCSettleInfo( + bytes.NewReader(settleData), + ) + if err != nil { + return nil, sql.NullTime{}, + fmt.Errorf("deserialize duplicate settle: %w", err) + } + + settleTime := normalizeTimeForSQL(settleInfo.SettleTime) + + return settleInfo.Preimage[:], sql.NullTime{ + Time: settleTime, + Valid: !settleTime.IsZero(), + }, nil +} + +// printMigrationSummary prints a summary of the migration. +func printMigrationSummary(stats *MigrationStats) { + if stats.TotalPayments == 0 { + log.Infof("No payments migrated - database is empty") + + return + } + + log.Infof("========================================") + log.Infof(" Payment Migration Summary") + log.Infof("========================================") + log.Infof("Total Payments: %d", stats.TotalPayments) + log.Infof(" Successful: %d", stats.SuccessfulPayments) + log.Infof(" Failed: %d", stats.FailedPayments) + log.Infof(" In-Flight: %d", stats.InFlightPayments) + log.Infof(" Initiated: %d", stats.InitiatedPayments) + log.Infof("") + log.Infof("Total HTLC Attempts: %d", stats.TotalAttempts) + log.Infof(" Settled: %d", stats.SettledAttempts) + log.Infof(" Failed: %d", stats.FailedAttempts) + log.Infof(" In-Flight: %d", stats.InFlightAttempts) + log.Infof("") + log.Infof("Total Route Hops: %d", stats.TotalHops) + + if stats.SkippedPayments > 0 { + log.Infof("") + log.Warnf("SKIPPED PAYMENTS:") + log.Warnf(" Indexed payments with missing buckets: %d", + stats.SkippedPayments) + log.Warnf(" These indicate minor DB inconsistencies.") + } + + if stats.DuplicatePayments > 0 { + log.Infof("") + log.Warnf("DUPLICATE PAYMENTS DETECTED:") + log.Warnf(" Unique payment hashes with duplicates: %d", + stats.DuplicatePayments) + log.Warnf(" Total duplicate entries migrated: %d", + stats.DuplicateEntries) + log.Warnf(" These were caused by an old LND bug.") + } + + log.Infof("") + log.Infof("Migration Duration: %v", stats.MigrationDuration) + log.Infof("========================================") +} diff --git a/payments/db/migration1/sql_migration_test.go b/payments/db/migration1/sql_migration_test.go new file mode 100644 index 000000000..ab0ee6cc1 --- /dev/null +++ b/payments/db/migration1/sql_migration_test.go @@ -0,0 +1,2794 @@ +//go:build test_db_postgres || test_db_sqlite + +package migration1 + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "sort" + "testing" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/record" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// TestMigrationKVToSQL tests the basic payment migration from KV to SQL. +func TestMigrationKVToSQL(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // Setup KV database and populate with test data. + kvDB := setupTestKVDB(t) + populateTestPayments(t, kvDB, 5) + + sqlStore := setupTestSQLDB(t) + + // Run migration in a single transaction. + err := runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) +} + +// TestMigrationSequenceOrder ensures the migration follows sequence order +// rather than lexicographic hash order. +func TestMigrationSequenceOrder(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + kvDB := setupTestKVDB(t) + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + var globalAttemptID uint64 + hash0 := [32]byte{} + hash1 := [32]byte{} + hash2 := [32]byte{} + hash0[0] = 3 + hash1[0] = 2 + hash2[0] = 1 + + if err := createTestPaymentInKV( + t, paymentsBucket, indexBucket, 0, hash0, + &globalAttemptID, + ); err != nil { + return err + } + + // We make sure that the duplicate payment is skipped because + // it will be migrated separately into payment_duplicates. + if err := createTestDuplicatePaymentWithIndex( + t, paymentsBucket, indexBucket, hash0, 1, false, + &globalAttemptID, + ); err != nil { + return err + } + if err := createTestPaymentInKV( + t, paymentsBucket, indexBucket, 2, hash1, + &globalAttemptID, + ); err != nil { + return err + } + if err := createTestPaymentInKV( + t, paymentsBucket, indexBucket, 3, hash2, + &globalAttemptID, + ); err != nil { + return err + } + + return nil + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + resp, err := sqlStore.QueryPayments(ctx, Query{ + MaxPayments: 10, + IncludeIncomplete: true, + }) + require.NoError(t, err) + require.Len(t, resp.Payments, 3) + + var ( + exp0 lntypes.Hash + exp1 lntypes.Hash + exp2 lntypes.Hash + ) + exp0[0] = 3 + exp1[0] = 2 + exp2[0] = 1 + + require.Equal(t, exp0, resp.Payments[0].Info.PaymentIdentifier) + require.Equal(t, exp1, resp.Payments[1].Info.PaymentIdentifier) + require.Equal(t, exp2, resp.Payments[2].Info.PaymentIdentifier) +} + +// TestMigrationDataIntegrity verifies that migrated payment data exactly +// matches the original KV data when fetched through the SQLStore +// (SQLStore.FetchPayment). This covers the SQLStore query path separately +// from the migration's own batch validation. +func TestMigrationDataIntegrity(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // Setup KV database with test data. + kvDB := setupTestKVDB(t) + numPayments := populateTestPayments(t, kvDB, 5) + + // Fetch all payments from KV before migration. + kvPayments := fetchAllPaymentsFromKV(t, kvDB) + require.Len(t, kvPayments, numPayments) + + // Setup SQL database and run migration. + sqlStore := setupTestSQLDB(t) + + err := runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Compare each KV payment with its SQL counterpart using deep equality. + // This ensures that ALL fields match, not just a few selected ones. + for _, kvPayment := range kvPayments { + comparePaymentData(t, ctx, sqlStore, kvPayment) + } +} + +// TestMigrationLegacyPayloadNormalized verifies that a payment whose HTLC +// hops carry LegacyPayload=true in the KV store is migrated and compared +// correctly. LegacyPayload was a hint used by the KV store to choose between +// the legacy and TLV hop-payload serialization formats. The SQL store does not +// serialize hop data at all — each field is stored natively in its own +// column — so the flag is never persisted there. normalizePaymentForCompare +// clears it on both sides before the equality check, so the comparison must +// succeed even when the KV source has LegacyPayload=true. +func TestMigrationLegacyPayloadNormalized(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + kvDB := setupTestKVDB(t) + + hash := createTestPaymentHash(t, 0) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket( + paymentsIndexBucket, + ) + if err != nil { + return err + } + + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists( + hash[:], + ) + if err != nil { + return err + } + + var paymentID lntypes.Hash + copy(paymentID[:], hash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(1000000), + CreationTime: time.Now().Add(-24 * time.Hour), + PaymentRequest: []byte("lnbc1test"), + } + + var b bytes.Buffer + if err = serializePaymentCreationInfo( + &b, creationInfo, + ); err != nil { + return err + } + if err = paymentBucket.Put( + paymentCreationInfoKey, b.Bytes(), + ); err != nil { + return err + } + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, 0) + if err = paymentBucket.Put( + paymentSequenceKey, seqBytes, + ); err != nil { + return err + } + + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + hop1Key, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + hop2Key, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + sessionKey, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + const attemptID = uint64(1) + attemptInfo := &HTLCAttemptInfo{ + AttemptID: attemptID, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: 500000, + TotalAmount: lnwire.MilliSatoshi(1000000), + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex( + hop1Key.PubKey(), + ), + ChannelID: 123456, + OutgoingTimeLock: 499500, + AmtToForward: 900000, + LegacyPayload: true, + }, + { + PubKeyBytes: NewVertex( + hop2Key.PubKey(), + ), + ChannelID: 789012, + OutgoingTimeLock: 499000, + AmtToForward: 800000, + LegacyPayload: true, + }, + }, + }, + AttemptTime: time.Now().Add(-2 * time.Hour), + Hash: &paymentID, + } + + if err = writeHTLCAttempt( + htlcBucket, attemptID, attemptInfo, + ); err != nil { + return err + } + + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(paymentID), + SettleTime: time.Now().Add(-1 * time.Hour), + } + if err = writeHTLCSettle( + htlcBucket, attemptID, settleInfo, + ); err != nil { + return err + } + + return createIndexEntry(indexBucket, seqBytes, hash) + }, func() {}) + require.NoError(t, err) + + // Fetch the payment from KV and verify LegacyPayload is true so we + // know the test data is correct before migration. + var kvPayment *MPPayment + err = kvdb.View(kvDB, func(tx kvdb.RTx) error { + bucket := tx.ReadBucket(paymentsRootBucket). + NestedReadBucket(hash[:]) + var err error + kvPayment, err = fetchPayment(bucket) + return err + }, func() {}) + require.NoError(t, err) + require.True(t, kvPayment.HTLCs[0].Route.Hops[0].LegacyPayload) + require.True(t, kvPayment.HTLCs[0].Route.Hops[1].LegacyPayload) + + sqlStore := setupTestSQLDB(t) + require.NoError(t, runPaymentsMigration(ctx, kvDB, sqlStore)) + + // comparePaymentData normalizes both sides (clearing LegacyPayload) + // before the equality check, so this must pass. + comparePaymentData(t, ctx, sqlStore, kvPayment) +} + +// TestMigrationWithDuplicates tests migration of duplicate payments into +// the payment_duplicates table. +func TestMigrationWithDuplicates(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // Setup KV database. + kvDB := setupTestKVDB(t) + + // Create a payment with duplicates. + hash := createTestPaymentHash(t, 0) + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + // Create root buckets. + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + // Create primary payment with sequence 0 and globally unique + // attempt ID. + var globalAttemptID uint64 + err = createTestPaymentInKV( + t, paymentsBucket, indexBucket, 0, hash, + &globalAttemptID, + ) + if err != nil { + return err + } + + // Add 2 duplicate payments for the same hash. + paymentBucket := paymentsBucket.NestedReadWriteBucket(hash[:]) + require.NotNil(t, paymentBucket) + + dupBucket, err := paymentBucket.CreateBucketIfNotExists( + duplicatePaymentsBucket, + ) + if err != nil { + return err + } + + // Create duplicate with sequence 1 using global attempt ID. + err = createTestDuplicatePayment( + t, dupBucket, hash, 1, true, &globalAttemptID, + ) + if err != nil { + return err + } + + // Create duplicate with sequence 2 using global attempt ID. + err = createTestDuplicatePayment( + t, dupBucket, hash, 2, false, &globalAttemptID, + ) + if err != nil { + return err + } + + return nil + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + // Run migration. + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Verify in SQL database. + var count int64 + err = sqlStore.db.ExecTx( + ctx, sqldb.ReadTxOpt(), func(q SQLQueries) error { + var err error + count, err = q.CountPayments(ctx) + return err + }, sqldb.NoOpReset, + ) + require.NoError(t, err) + require.Equal( + t, int64(1), count, "SQL DB should have 1 payment", + ) + + var ( + dbPayment sqlc.FetchPaymentRow + duplicates []sqlc.PaymentDuplicate + ) + err = sqlStore.db.ExecTx( + ctx, sqldb.ReadTxOpt(), func(q SQLQueries) error { + var err error + dbPayment, err = q.FetchPayment(ctx, hash[:]) + if err != nil { + return err + } + + duplicates, err = q.FetchPaymentDuplicates( + ctx, dbPayment.Payment.ID, + ) + return err + }, sqldb.NoOpReset, + ) + require.NoError(t, err) + + require.Len(t, duplicates, 2) + sort.SliceStable(duplicates, func(i, j int) bool { + return duplicates[i].AmountMsat < duplicates[j].AmountMsat + }) + + require.Equal(t, int64(2001000), duplicates[0].AmountMsat) + require.False(t, duplicates[0].FailReason.Valid) + require.NotEmpty(t, duplicates[0].SettlePreimage) + + require.Equal(t, int64(2002000), duplicates[1].AmountMsat) + require.True(t, duplicates[1].FailReason.Valid) + require.Equal( + t, int32(FailureReasonError), + duplicates[1].FailReason.Int32, + ) + require.Empty(t, duplicates[1].SettlePreimage) +} + +// TestDuplicatePaymentsWithoutAttemptInfo verifies duplicate payments without +// attempt info are migrated with terminal failure reasons. +func TestDuplicatePaymentsWithoutAttemptInfo(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + hash := createTestPaymentHash(t, 0) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + var globalAttemptID uint64 + err = createTestPaymentInKV( + t, paymentsBucket, indexBucket, 1, hash, + &globalAttemptID, + ) + if err != nil { + return err + } + + paymentBucket := paymentsBucket.NestedReadWriteBucket( + hash[:], + ) + require.NotNil(t, paymentBucket) + + dupBucket, err := paymentBucket.CreateBucketIfNotExists( + duplicatePaymentsBucket, + ) + if err != nil { + return err + } + + if err := createDuplicateWithoutAttemptInfo( + t, dupBucket, hash, 2, true, false, + ); err != nil { + return err + } + if err := createDuplicateWithoutAttemptInfo( + t, dupBucket, hash, 3, false, true, + ); err != nil { + return err + } + if err := createDuplicateWithoutAttemptInfo( + t, dupBucket, hash, 4, false, false, + ); err != nil { + return err + } + + return nil + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + var ( + dbPayment sqlc.FetchPaymentRow + duplicates []sqlc.PaymentDuplicate + ) + err = sqlStore.db.ExecTx( + ctx, sqldb.ReadTxOpt(), func(q SQLQueries) error { + var err error + dbPayment, err = q.FetchPayment(ctx, hash[:]) + if err != nil { + return err + } + + duplicates, err = q.FetchPaymentDuplicates( + ctx, dbPayment.Payment.ID, + ) + return err + }, sqldb.NoOpReset, + ) + require.NoError(t, err) + + require.Len(t, duplicates, 3) + sort.SliceStable(duplicates, func(i, j int) bool { + return duplicates[i].AmountMsat < duplicates[j].AmountMsat + }) + + require.Equal(t, int64(2002000), duplicates[0].AmountMsat) + require.NotEmpty(t, duplicates[0].SettlePreimage) + require.False(t, duplicates[0].FailReason.Valid) + + require.Equal(t, int64(2003000), duplicates[1].AmountMsat) + require.True(t, duplicates[1].FailReason.Valid) + require.Equal( + t, int32(FailureReasonNoRoute), + duplicates[1].FailReason.Int32, + ) + require.Empty(t, duplicates[1].SettlePreimage) + + require.Equal(t, int64(2004000), duplicates[2].AmountMsat) + require.True(t, duplicates[2].FailReason.Valid) + require.Equal( + t, int32(FailureReasonError), + duplicates[2].FailReason.Int32, + ) + require.Empty(t, duplicates[2].SettlePreimage) +} + +// TestMigratePaymentWithMPP tests migration of a payment with MPP (multi-path +// payment) records. +func TestMigratePaymentWithMPP(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + // Create a payment with MPP. + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_mpp_payment_hash_12345")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithMPP( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + // Run migration. + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Verify payment matches. + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithAMP tests migration of a payment with AMP (atomic +// multi-path) records. +func TestMigratePaymentWithAMP(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_amp_payment_hash_12345")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithAMP( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithAMPSignedChildIndex tests migration of an AMP payment +// where the child index has the signed bit set. +func TestMigratePaymentWithAMPSignedChildIndex(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_amp_child_idx_8000")) + + const childIndex = uint32(0x80000001) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithAMPChildIndex( + t, paymentsBucket, indexBucket, paymentHash, childIndex, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithCustomRecords tests migration of a payment with custom +// records. +func TestMigratePaymentWithCustomRecords(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_custom_records_hash_12")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithCustomRecords( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithBlindedRoute tests migration of a payment with blinded +// route. +func TestMigratePaymentWithBlindedRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_blinded_route_hash_123")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithBlindedRoute( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithMetadata tests migration of a payment with hop +// metadata. +func TestMigratePaymentWithMetadata(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_metadata_payment_hash_")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithMetadata( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentWithAllFeatures tests migration with all optional +// features enabled. +func TestMigratePaymentWithAllFeatures(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_all_features_hash_1234")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createPaymentWithAllFeatures( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// TestMigratePaymentFeatureCombinations tests selected feature combinations +// in a single migration to cover interactions without random data. +func TestMigratePaymentFeatureCombinations(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + cases := []paymentFeatureSet{ + { + name: "mpp_custom", + mpp: true, + customRecords: true, + }, + { + name: "amp_blinded", + amp: true, + blindedRoute: true, + }, + { + name: "custom_metadata", + customRecords: true, + hopMetadata: true, + }, + { + name: "blinded_metadata", + blindedRoute: true, + hopMetadata: true, + }, + { + name: "mpp_metadata", + mpp: true, + hopMetadata: true, + }, + } + + hashes := make([][32]byte, 0, len(cases)) + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + var globalAttemptID uint64 + for i, c := range cases { + hash := sha256.Sum256([]byte(c.name)) + hashes = append(hashes, hash) + + err := createPaymentWithFeatureSet( + t, paymentsBucket, indexBucket, hash, + uint64(10+i), c, &globalAttemptID, + ) + if err != nil { + return err + } + } + + return nil + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + for _, hash := range hashes { + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, hash) + } +} + +// TestMigratePaymentWithFailureMessage tests migration of a payment with a +// failed HTLC that includes a failure message. +func TestMigratePaymentWithFailureMessage(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_fail_msg_hash_123456789")) + + // Create a payment with a failed HTLC. + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + // Create payment bucket. + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists( + paymentHash[:], + ) + if err != nil { + return err + } + + // Add creation info. + var paymentID lntypes.Hash + copy(paymentID[:], paymentHash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(1000000), + CreationTime: time.Now().Add(-24 * time.Hour), + PaymentRequest: []byte("lnbc10utest"), + } + + // Use a separate buffer for payment creation info to avoid + // reuse issues when serializing HTLC attempts later. + var creationInfoBuf bytes.Buffer + err = serializePaymentCreationInfo( + &creationInfoBuf, creationInfo, + ) + if err != nil { + return err + } + + serialized := creationInfoBuf.Bytes() + + err = paymentBucket.Put( + paymentCreationInfoKey, serialized, + ) + if err != nil { + return err + } + + // Add sequence number. + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, 50) + err = paymentBucket.Put(paymentSequenceKey, seqBytes) + if err != nil { + return err + } + + // Add payment-level failure reason. + failReasonBytes := []byte{byte(FailureReasonNoRoute)} + err = paymentBucket.Put( + paymentFailInfoKey, failReasonBytes, + ) + if err != nil { + return err + } + + // Create HTLC bucket with one failed attempt. + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + // Create the failed attempt with a failure message. + attemptID := uint64(500) + sessionKey, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + hopKey, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + // Create a proper copy of the hash instead of referencing + // the local variable directly. + attemptHash := new(lntypes.Hash) + copy(attemptHash[:], paymentHash[:]) + + //nolint:ll + attemptInfo := &HTLCAttemptInfo{ + AttemptID: attemptID, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: 500000, + TotalAmount: 900, + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex(hopKey.PubKey()), + ChannelID: 12345, + OutgoingTimeLock: 499500, + AmtToForward: 850, + }, + }, + }, + AttemptTime: time.Now().Add(-2 * time.Hour), + Hash: attemptHash, + } + + // Write attempt info. + attemptKey := make([]byte, len(htlcAttemptInfoKey)+8) + copy(attemptKey, htlcAttemptInfoKey) + byteOrder.PutUint64( + attemptKey[len(htlcAttemptInfoKey):], attemptID, + ) + + var b bytes.Buffer + err = serializeHTLCAttemptInfo(&b, attemptInfo) + if err != nil { + return err + } + err = htlcBucket.Put(attemptKey, b.Bytes()) + if err != nil { + return err + } + + // Add failure info with a message. + //nolint:ll + failInfo := &HTLCFailInfo{ + FailTime: time.Now().Add(-1 * time.Hour), + Message: &lnwire.FailTemporaryChannelFailure{}, + Reason: HTLCFailMessage, + FailureSourceIndex: 1, + } + + failKey := make([]byte, len(htlcFailInfoKey)+8) + copy(failKey, htlcFailInfoKey) + byteOrder.PutUint64(failKey[len(htlcFailInfoKey):], attemptID) + + b.Reset() + if err := serializeHTLCFailInfo(&b, failInfo); err != nil { + return err + } + if err := htlcBucket.Put(failKey, b.Bytes()); err != nil { + return err + } + + // Create index entry. + var idx bytes.Buffer + if err := WriteElements( + &idx, paymentIndexTypeHash, paymentHash[:], + ); err != nil { + return err + } + return indexBucket.Put(seqBytes, idx.Bytes()) + }, func() {}) + require.NoError(t, err) + + // Migrate to SQL. + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Verify data matches. + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) +} + +// setupTestKVDB creates a temporary KV database for testing. +func setupTestKVDB(t *testing.T) kvdb.Backend { + t.Helper() + + backend, cleanup, err := kvdb.GetTestBackend(t.TempDir(), "payments") + require.NoError(t, err) + t.Cleanup(cleanup) + + return backend +} + +// populateTestPayments populates the KV database with test payment data. +func populateTestPayments(t *testing.T, db kvdb.Backend, numPayments int) int { + t.Helper() + + err := kvdb.Update(db, func(tx kvdb.RwTx) error { + // Create root buckets. + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + // Create test payments with globally unique attempt IDs. + var globalAttemptID uint64 + for i := 0; i < numPayments; i++ { + hash := createTestPaymentHash(t, i) + + err := createTestPaymentInKV( + t, paymentsBucket, indexBucket, uint64(i), hash, + &globalAttemptID, + ) + if err != nil { + return err + } + } + + return nil + }, func() {}) + + require.NoError(t, err) + return numPayments +} + +// serializeDuplicatePaymentCreationInfo serializes PaymentCreationInfo for +// duplicate payments. The time is stored in seconds (not nanoseconds) to match +// the format used by deserializeDuplicatePaymentCreationInfo in the KV store. +func serializeDuplicatePaymentCreationInfo(w io.Writer, + c *PaymentCreationInfo) error { + + var scratch [8]byte + + if _, err := w.Write(c.PaymentIdentifier[:]); err != nil { + return err + } + + byteOrder.PutUint64(scratch[:], uint64(c.Value)) + if _, err := w.Write(scratch[:]); err != nil { + return err + } + + // Store time in seconds (not nanoseconds) for duplicate payments. + // This matches the deserialization format used in + // deserializeDuplicatePaymentCreationInfo. + var unixSec int64 + if !c.CreationTime.IsZero() { + unixSec = c.CreationTime.Unix() + } + byteOrder.PutUint64(scratch[:], uint64(unixSec)) + if _, err := w.Write(scratch[:]); err != nil { + return err + } + + byteOrder.PutUint32(scratch[:4], uint32(len(c.PaymentRequest))) + if _, err := w.Write(scratch[:4]); err != nil { + return err + } + + if _, err := w.Write(c.PaymentRequest); err != nil { + return err + } + + return nil +} + +// createTestPaymentHash creates a deterministic payment hash for testing. +func createTestPaymentHash(t *testing.T, seed int) [32]byte { + t.Helper() + + hash := sha256.Sum256([]byte{byte(seed)}) + return hash +} + +// createTestPaymentInKV creates a single payment in the KV store. +func createTestPaymentInKV(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, seqNum uint64, hash [32]byte, + globalAttemptID *uint64) error { + + t.Helper() + + // Create payment bucket. + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists(hash[:]) + if err != nil { + return err + } + + // Create payment creation info. + var paymentID lntypes.Hash + copy(paymentID[:], hash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(1000000 + seqNum*1000), + CreationTime: time.Now().Add(-24 * time.Hour), + PaymentRequest: []byte("lnbc1test"), + } + + // Serialize and write creation info. + var b bytes.Buffer + err = serializePaymentCreationInfo(&b, creationInfo) + if err != nil { + return err + } + err = paymentBucket.Put(paymentCreationInfoKey, b.Bytes()) + if err != nil { + return err + } + + // Store sequence number. + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, seqNum) + err = paymentBucket.Put(paymentSequenceKey, seqBytes) + if err != nil { + return err + } + + // Add one HTLC attempt for each payment with globally unique ID. + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + // Increment global attempt ID and create HTLC attempt. So we have a + // globally unique attempt ID for the HTLC attempt. + *globalAttemptID++ + err = createTestHTLCAttempt( + t, htlcBucket, hash, *globalAttemptID, seqNum%3 == 0, + ) + if err != nil { + return err + } + + var idx bytes.Buffer + err = WriteElements(&idx, paymentIndexTypeHash, hash[:]) + if err != nil { + return err + } + + return indexBucket.Put(seqBytes, idx.Bytes()) +} + +// createTestHTLCAttempt creates a test HTLC attempt in the KV store. +func createTestHTLCAttempt(t *testing.T, htlcBucket kvdb.RwBucket, + paymentHash [32]byte, attemptID uint64, shouldSettle bool) error { + t.Helper() + + // Generate a session key. + sessionKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + // Create a simple 2-hop route. + hop1Key, err := btcec.NewPrivateKey() + require.NoError(t, err) + + hop2Key, err := btcec.NewPrivateKey() + require.NoError(t, err) + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + // Convert session key to [32]byte. + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + attemptInfo := &HTLCAttemptInfo{ + AttemptID: attemptID, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: 500000, + TotalAmount: 900, + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex( + hop1Key.PubKey(), + ), + ChannelID: 12345, + OutgoingTimeLock: 499500, + AmtToForward: 850, + }, + { + PubKeyBytes: NewVertex( + hop2Key.PubKey(), + ), + ChannelID: 67890, + OutgoingTimeLock: 499000, + AmtToForward: 800, + }, + }, + }, + AttemptTime: time.Now().Add(-2 * time.Hour), + Hash: (*lntypes.Hash)(&paymentHash), + } + + // Serialize and write attempt info. + attemptKey := make([]byte, len(htlcAttemptInfoKey)+8) + copy(attemptKey, htlcAttemptInfoKey) + byteOrder.PutUint64(attemptKey[len(htlcAttemptInfoKey):], attemptID) + + var b bytes.Buffer + err = serializeHTLCAttemptInfo(&b, attemptInfo) + if err != nil { + return err + } + err = htlcBucket.Put(attemptKey, b.Bytes()) + if err != nil { + return err + } + + // Add settlement if requested. + if shouldSettle { + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(paymentHash), + SettleTime: time.Now().Add(-1 * time.Hour), + } + + settleKey := make([]byte, len(htlcSettleInfoKey)+8) + copy(settleKey, htlcSettleInfoKey) + byteOrder.PutUint64( + settleKey[len(htlcSettleInfoKey):], attemptID, + ) + + var sb bytes.Buffer + err = serializeHTLCSettleInfo(&sb, settleInfo) + if err != nil { + return err + } + err = htlcBucket.Put(settleKey, sb.Bytes()) + if err != nil { + return err + } + } + + return nil +} + +// createTestDuplicatePaymentWithIndex creates a duplicate payment and adds +// a matching entry into the global payment sequence index. +func createTestDuplicatePaymentWithIndex(t *testing.T, + paymentsBucket kvdb.RwBucket, indexBucket kvdb.RwBucket, + paymentHash [32]byte, seqNum uint64, shouldSettle bool, + globalAttemptID *uint64) error { + t.Helper() + + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists( + paymentHash[:], + ) + if err != nil { + return err + } + + dupBucket, err := paymentBucket.CreateBucketIfNotExists( + duplicatePaymentsBucket, + ) + if err != nil { + return err + } + + if err := createTestDuplicatePayment( + t, dupBucket, paymentHash, seqNum, shouldSettle, + globalAttemptID, + ); err != nil { + return err + } + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, seqNum) + var idx bytes.Buffer + if err := WriteElements( + &idx, paymentIndexTypeHash, paymentHash[:], + ); err != nil { + return err + } + + return indexBucket.Put(seqBytes, idx.Bytes()) +} + +// createTestDuplicatePayment creates a duplicate payment in the KV store. +func createTestDuplicatePayment(t *testing.T, + dupBucket kvdb.RwBucket, paymentHash [32]byte, seqNum uint64, + shouldSettle bool, globalAttemptID *uint64) error { + + t.Helper() + + // Create bucket for this duplicate using sequence number as key. + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, seqNum) + + dupPaymentBucket, err := dupBucket.CreateBucketIfNotExists(seqBytes) + if err != nil { + return err + } + + // Store sequence number. + err = dupPaymentBucket.Put(duplicatePaymentSequenceKey, seqBytes) + if err != nil { + return err + } + + // Create payment creation info. + var paymentID lntypes.Hash + copy(paymentID[:], paymentHash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(2000000 + seqNum*1000), + CreationTime: time.Now().Add(-48 * time.Hour), + PaymentRequest: []byte("lnbc1duplicate"), + } + + var b bytes.Buffer + err = serializeDuplicatePaymentCreationInfo(&b, creationInfo) + if err != nil { + return err + } + err = dupPaymentBucket.Put(duplicatePaymentCreationInfoKey, b.Bytes()) + if err != nil { + return err + } + + // Generate a session key for the duplicate attempt. + sessionKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + // Create route for duplicate. + hop1Key, err := btcec.NewPrivateKey() + require.NoError(t, err) + + hop2Key, err := btcec.NewPrivateKey() + require.NoError(t, err) + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + // Use globally unique attempt ID. + *globalAttemptID++ + attemptID := *globalAttemptID + + duplicateAttempt := &duplicateHTLCAttemptInfo{ + attemptID: attemptID, + sessionKey: sessionKeyBytes, + route: Route{ + TotalTimeLock: 500000, + TotalAmount: 900, + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex( + hop1Key.PubKey(), + ), + ChannelID: 12345, + OutgoingTimeLock: 499500, + AmtToForward: 850, + }, + { + PubKeyBytes: NewVertex( + hop2Key.PubKey(), + ), + ChannelID: 67890, + OutgoingTimeLock: 499000, + AmtToForward: 800, + }, + }, + }, + } + + // Serialize and write attempt info (using existing WriteElements + // and SerializeRoute). + var ab bytes.Buffer + if err := WriteElements( + &ab, duplicateAttempt.attemptID, + duplicateAttempt.sessionKey, + ); err != nil { + return err + } + if err := SerializeRoute(&ab, duplicateAttempt.route); err != nil { + return err + } + err = dupPaymentBucket.Put(duplicatePaymentAttemptInfoKey, ab.Bytes()) + if err != nil { + return err + } + + // Add settlement if requested. + if shouldSettle { + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(paymentHash), + SettleTime: time.Now().Add(-1 * time.Hour), + } + + var sb bytes.Buffer + err = serializeHTLCSettleInfo(&sb, settleInfo) + if err != nil { + return err + } + err = dupPaymentBucket.Put( + duplicatePaymentSettleInfoKey, sb.Bytes(), + ) + if err != nil { + return err + } + } + + return nil +} + +// createDuplicateWithoutAttemptInfo creates a duplicate payment bucket with +// settle/fail info but without attempt info. +func createDuplicateWithoutAttemptInfo(t *testing.T, + dupBucket kvdb.RwBucket, paymentHash [32]byte, seqNum uint64, + shouldSettle bool, shouldFail bool) error { + t.Helper() + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, seqNum) + + dupPaymentBucket, err := dupBucket.CreateBucketIfNotExists(seqBytes) + if err != nil { + return err + } + + if err := dupPaymentBucket.Put( + duplicatePaymentSequenceKey, seqBytes, + ); err != nil { + return err + } + + var paymentID lntypes.Hash + copy(paymentID[:], paymentHash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(2000000 + seqNum*1000), + CreationTime: time.Now().Add(-48 * time.Hour), + PaymentRequest: []byte("lnbc1duplicate"), + } + + var b bytes.Buffer + err = serializeDuplicatePaymentCreationInfo(&b, creationInfo) + if err != nil { + return err + } + if err := dupPaymentBucket.Put( + duplicatePaymentCreationInfoKey, b.Bytes(), + ); err != nil { + return err + } + + switch { + case shouldSettle && shouldFail: + return fmt.Errorf("invalid duplicate state") + case shouldSettle: + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(paymentHash), + SettleTime: time.Now().Add(-1 * time.Hour), + } + + var sb bytes.Buffer + err = serializeHTLCSettleInfo(&sb, settleInfo) + if err != nil { + return err + } + if err := dupPaymentBucket.Put( + duplicatePaymentSettleInfoKey, sb.Bytes(), + ); err != nil { + return err + } + case shouldFail: + failReasonBytes := []byte{byte(FailureReasonNoRoute)} + if err := dupPaymentBucket.Put( + duplicatePaymentFailInfoKey, failReasonBytes, + ); err != nil { + return err + } + } + + return nil +} + +// fetchAllPaymentsFromKV fetches all payments from the KV store using the +// KVStore implementation. +func fetchAllPaymentsFromKV(t *testing.T, kvDB kvdb.Backend) []*MPPayment { + t.Helper() + + kvStore, err := NewKVStore(kvDB, WithNoMigration(true)) + require.NoError(t, err) + + payments, err := kvStore.FetchPayments() + require.NoError(t, err) + + return payments +} + +// normalizePaymentData makes sure that the payment data is normalized for +// comparison using the same logic as in-migration validation. +func normalizePaymentData(payment *MPPayment) { + normalizePaymentForCompare(payment) +} + +// comparePaymentData compares a KV payment with its SQL counterpart using +// deep equality check (similar to invoice migration). +func comparePaymentData(t *testing.T, ctx context.Context, sqlStore *SQLStore, + kvPayment *MPPayment) { + + t.Helper() + + // Fetch the SQL payment as MPPayment using SQLStore. + var paymentHash lntypes.Hash + copy(paymentHash[:], kvPayment.Info.PaymentIdentifier[:]) + + sqlPayment, err := sqlStore.FetchPayment(ctx, paymentHash) + require.NoError(t, err, "SQL payment should exist for %x", + paymentHash[:8]) + + // Normalize time precision to microseconds. + normalizePaymentData(kvPayment) + normalizePaymentData(sqlPayment) + + // Deep equality check - compares all fields recursively. + require.Equal(t, kvPayment, sqlPayment, + "KV and SQL payments should be equal for %x", paymentHash[:8]) +} + +// runPaymentsMigration executes the payment migration from KV to SQL within +// a SQL transaction. +func runPaymentsMigration(ctx context.Context, kvDB kvdb.Backend, + sqlStore *SQLStore) error { + + return sqlStore.db.ExecTx( + ctx, sqldb.WriteTxOpt(), func(tx SQLQueries) error { + migTx, ok := tx.(SQLMigrationQueries) + if !ok { + return fmt.Errorf("db does not implement " + + "SQLMigrationQueries") + } + + return MigratePaymentsKVToSQL( + ctx, kvDB, migTx, &SQLStoreConfig{ + QueryCfg: sqlStore.cfg.QueryCfg, + }, + ) + }, sqldb.NoOpReset, + ) +} + +// assertPaymentDataMatches verifies a payment in KV matches its SQL counterpart +// using deep equality check. +func assertPaymentDataMatches(t *testing.T, ctx context.Context, + kvDB kvdb.Backend, sqlStore *SQLStore, hash [32]byte) { + t.Helper() + + // Fetch from KV. + var kvPayment *MPPayment + err := kvdb.View(kvDB, func(tx kvdb.RTx) error { + paymentsBucket := tx.ReadBucket(paymentsRootBucket) + if paymentsBucket == nil { + return nil + } + + paymentBucket := paymentsBucket.NestedReadBucket(hash[:]) + if paymentBucket == nil { + return nil + } + + var err error + kvPayment, err = fetchPayment(paymentBucket) + return err + }, func() {}) + require.NoError(t, err) + + if kvPayment == nil { + // Payment doesn't exist in KV, should not exist in SQL + // either. + var paymentHash lntypes.Hash + copy(paymentHash[:], hash[:]) + _, err := sqlStore.FetchPayment(ctx, paymentHash) + require.Error( + t, err, "payment should not exist in SQL if not "+ + "in KV", + ) + return + } + + // Use the deep comparison function. + comparePaymentData(t, ctx, sqlStore, kvPayment) +} + +// paymentTestConfig holds configuration for creating test payments with various +// features. +type paymentTestConfig struct { + hash [32]byte + seqNum uint64 + value lnwire.MilliSatoshi + creationTime time.Time + paymentRequest string + attemptID uint64 + numHops int + baseChannelID uint64 + baseTimeLock uint32 + paymentCustomRecs lnwire.CustomRecords + attemptCustomRecs lnwire.CustomRecords + hopConfigurator func(hop *Hop, index int, isFinal bool) +} + +// serializeAndPut serializes data using the provided serializer function and +// writes it to the bucket. +func serializeAndPut(bucket kvdb.RwBucket, key []byte, + serializer func(io.Writer) error) error { + + var b bytes.Buffer + if err := serializer(&b); err != nil { + return err + } + return bucket.Put(key, b.Bytes()) +} + +// generateSessionKey creates a new session key and returns the private key, +// source public key vertex, and serialized key bytes. +func generateSessionKey(t *testing.T) (*btcec.PrivateKey, Vertex, + [32]byte, error) { + + t.Helper() + + sessionKey, err := btcec.NewPrivateKey() + if err != nil { + return nil, Vertex{}, [32]byte{}, err + } + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + return sessionKey, sourcePubKey, sessionKeyBytes, nil +} + +// createTestHops creates the specified number of test hops with the given +// parameters. The configurator function is called for each hop to allow +// feature-specific customization. +func createTestHops(t *testing.T, numHops int, baseAmount lnwire.MilliSatoshi, + baseChannelID uint64, baseTimeLock uint32, + configurator func(*Hop, int, bool)) ([]*Hop, + lnwire.MilliSatoshi, error) { + + t.Helper() + + hops := make([]*Hop, numHops) + currentAmt := baseAmount + + for i := 0; i < numHops; i++ { + hopKey, err := btcec.NewPrivateKey() + if err != nil { + return nil, 0, err + } + + amt := baseAmount - lnwire.MilliSatoshi(uint64(i)*100) + hop := &Hop{ + PubKeyBytes: NewVertex(hopKey.PubKey()), + ChannelID: baseChannelID + uint64(i), + OutgoingTimeLock: baseTimeLock - uint32(i*40), + AmtToForward: amt, + } + + // Apply feature-specific configuration. + if configurator != nil { + configurator(hop, i, i == numHops-1) + } + + hops[i] = hop + currentAmt = amt + } + + return hops, currentAmt, nil +} + +// writeHTLCAttempt writes the HTLC attempt info to the bucket. +func writeHTLCAttempt(bucket kvdb.RwBucket, attemptID uint64, + info *HTLCAttemptInfo) error { + + attemptKey := make([]byte, len(htlcAttemptInfoKey)+8) + copy(attemptKey, htlcAttemptInfoKey) + byteOrder.PutUint64(attemptKey[len(htlcAttemptInfoKey):], attemptID) + + return serializeAndPut(bucket, attemptKey, func(w io.Writer) error { + return serializeHTLCAttemptInfo(w, info) + }) +} + +// writeHTLCSettle writes the HTLC settle info to the bucket. +func writeHTLCSettle(bucket kvdb.RwBucket, attemptID uint64, + info *HTLCSettleInfo) error { + + settleKey := make([]byte, len(htlcSettleInfoKey)+8) + copy(settleKey, htlcSettleInfoKey) + byteOrder.PutUint64(settleKey[len(htlcSettleInfoKey):], attemptID) + + return serializeAndPut(bucket, settleKey, func(w io.Writer) error { + return serializeHTLCSettleInfo(w, info) + }) +} + +// createIndexEntry creates a payment index entry in the index bucket. +func createIndexEntry(indexBucket kvdb.RwBucket, seqBytes []byte, + hash [32]byte) error { + + var idx bytes.Buffer + if err := WriteElements( + &idx, paymentIndexTypeHash, hash[:], + ); err != nil { + return err + } + + return indexBucket.Put(seqBytes, idx.Bytes()) +} + +// createTestPayment creates a test payment with the specified configuration, +// handling all common boilerplate code. +func createTestPayment(t *testing.T, paymentsBucket, indexBucket kvdb.RwBucket, + cfg paymentTestConfig) error { + + t.Helper() + + // Create payment bucket. + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists( + cfg.hash[:], + ) + if err != nil { + return err + } + + // Create payment ID. + var paymentID lntypes.Hash + copy(paymentID[:], cfg.hash[:]) + + // Create and serialize payment creation info. + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: cfg.value, + CreationTime: cfg.creationTime, + PaymentRequest: []byte(cfg.paymentRequest), + FirstHopCustomRecords: cfg.paymentCustomRecs, + } + + err = serializeAndPut( + paymentBucket, paymentCreationInfoKey, + func(w io.Writer) error { + return serializePaymentCreationInfo(w, creationInfo) + }, + ) + if err != nil { + return err + } + + // Store sequence number. + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, cfg.seqNum) + if err := paymentBucket.Put(paymentSequenceKey, seqBytes); err != nil { + return err + } + + // Create HTLC bucket. + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + // Generate session key. + _, sourcePubKey, sessionKeyBytes, err := generateSessionKey(t) + if err != nil { + return err + } + + // Create route with hops. + hops, totalAmount, err := createTestHops( + t, cfg.numHops, cfg.value, cfg.baseChannelID, + cfg.baseTimeLock, cfg.hopConfigurator, + ) + if err != nil { + return err + } + + // Create and serialize attempt info. + attemptInfo := &HTLCAttemptInfo{ + AttemptID: cfg.attemptID, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: cfg.baseTimeLock, + TotalAmount: totalAmount, + SourcePubKey: sourcePubKey, + Hops: hops, + FirstHopWireCustomRecords: cfg.attemptCustomRecs, + }, + AttemptTime: cfg.creationTime.Add(time.Minute), + Hash: (*lntypes.Hash)(&cfg.hash), + } + + if err = writeHTLCAttempt( + htlcBucket, cfg.attemptID, attemptInfo, + ); err != nil { + return err + } + + // Add settlement. + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(cfg.hash), + SettleTime: cfg.creationTime.Add(2 * time.Minute), + } + + if err := writeHTLCSettle( + htlcBucket, cfg.attemptID, settleInfo, + ); err != nil { + return err + } + + // Create index entry. + return createIndexEntry(indexBucket, seqBytes, cfg.hash) +} + +// createPaymentWithMPP creates a payment with MPP records on the final hop. +func createPaymentWithMPP(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 1, + value: 50000, + creationTime: time.Date( + 2024, 1, 1, 12, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc500n1test_mpp", + attemptID: 1, + numHops: 3, + baseChannelID: 100000, + baseTimeLock: 500000, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + if isFinal { + var paymentAddr [32]byte + copy( + paymentAddr[:], + []byte( + "test_mpp_payment_"+ + "address_32", + ), + ) + hop.MPP = record.NewMPP( + lnwire.MilliSatoshi(50000), + paymentAddr, + ) + } + }, + }, + ) +} + +// createPaymentWithAMP creates a payment with AMP records on the final hop. +func createPaymentWithAMP(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + t.Helper() + + return createPaymentWithAMPChildIndex( + t, paymentsBucket, indexBucket, hash, 0, + ) +} + +// createPaymentWithAMPChildIndex creates a payment with AMP records on the +// final hop and a specific child index. +func createPaymentWithAMPChildIndex(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte, childIndex uint32) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 2, + value: 75000, + creationTime: time.Date( + 2024, 2, 1, 10, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc750n1test_amp", + attemptID: 1, + numHops: 2, + baseChannelID: 200000, + baseTimeLock: 600000, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + if isFinal { + var rootShare [32]byte + copy( + rootShare[:], + []byte( + "test_amp_root_share"+ + "_12345678", + ), + ) + var setID [32]byte + copy( + setID[:], + []byte( + "test_amp_set_id_"+ + "123456789012", + ), + ) + hop.AMP = record.NewAMP( + rootShare, setID, childIndex, + ) + } + }, + }, + ) +} + +// createPaymentWithCustomRecords creates a payment with custom records at all +// levels. +func createPaymentWithCustomRecords(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 3, + value: 100000, + creationTime: time.Date( + 2024, 3, 1, 14, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc1m1test_custom", + attemptID: 1, + numHops: 3, + baseChannelID: 300000, + baseTimeLock: 700000, + paymentCustomRecs: lnwire.CustomRecords{ + 65536: []byte("payment_level_value_1"), + 65537: []byte("payment_level_value_2"), + }, + attemptCustomRecs: lnwire.CustomRecords{ + 65541: []byte("attempt_custom_value_1"), + 65542: []byte("attempt_custom_value_2"), + }, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + hop.CustomRecords = record.CustomSet{ + 65538 + uint64(index): []byte( + fmt.Sprintf( + "hop_%d_custom_value", + index, + ), + ), + } + }, + }, + ) +} + +// createPaymentWithBlindedRoute creates a payment with blinded route data. +func createPaymentWithBlindedRoute(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 4, + value: 120000, + creationTime: time.Date( + 2024, 4, 1, 16, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc1200n1test_blinded", + attemptID: 1, + numHops: 4, + baseChannelID: 400000, + baseTimeLock: 800000, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + if isFinal { + blindingKey, err := btcec. + NewPrivateKey() + + require.NoError(t, err) + + hop.BlindingPoint = blindingKey.PubKey() + hop.EncryptedData = []byte( + "encrypted_blinded_route_" + + "data_test_value_12345", + ) + hop.TotalAmtMsat = lnwire.MilliSatoshi( + 119400, + ) + } + }, + }, + ) +} + +// createPaymentWithMetadata creates a payment with hop metadata. +func createPaymentWithMetadata(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 5, + value: 80000, + creationTime: time.Date( + 2024, 5, 1, 18, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc800n1test_metadata", + attemptID: 1, + numHops: 3, + baseChannelID: 500000, + baseTimeLock: 900000, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + hop.Metadata = []byte( + fmt.Sprintf( + "hop_%d_metadata_value", index, + ), + ) + }, + }, + ) +} + +// createPaymentWithAllFeatures creates a payment with all optional features +// enabled. +func createPaymentWithAllFeatures(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + return createTestPayment( + t, paymentsBucket, indexBucket, + paymentTestConfig{ + hash: hash, + seqNum: 6, + value: 150000, + creationTime: time.Date( + 2024, 6, 1, 20, 0, 0, 0, time.UTC, + ), + paymentRequest: "lnbc1500n1test_all_features", + attemptID: 1, + numHops: 4, + baseChannelID: 600000, + baseTimeLock: 1000000, + paymentCustomRecs: lnwire.CustomRecords{ + 65543: []byte("all_features_payment_custom_1"), + 65544: []byte("all_features_payment_custom_2"), + }, + attemptCustomRecs: lnwire.CustomRecords{ + 65549: []byte("all_feat_attempt_custom_1"), + 65550: []byte("all_feat_attempt_custom_2"), + }, + hopConfigurator: func(hop *Hop, index int, + isFinal bool) { + + // Add custom records and metadata to all hops. + hop.CustomRecords = record.CustomSet{ + 65545 + uint64(index): []byte( + fmt.Sprintf( + "all_feat_hop_%d", + index, + ), + ), + } + hop.Metadata = []byte( + fmt.Sprintf( + "all_feat_metadata_%d", index, + ), + ) + + // Add MPP and blinded route data to final hop. + if isFinal { + var paymentAddr [32]byte + copy( + paymentAddr[:], + []byte( + "all_features_mpp_"+ + "addr_123456", + ), + ) + hop.MPP = record.NewMPP( + lnwire.MilliSatoshi(149250), + paymentAddr, + ) + + blindingKey, err := btcec. + NewPrivateKey() + + require.NoError(t, err) + hop.BlindingPoint = blindingKey.PubKey() + hop.EncryptedData = []byte( + "all_features_encrypted_" + + "blinded_data_123456", + ) + hop.TotalAmtMsat = lnwire.MilliSatoshi( + 149250, + ) + } + }, + }, + ) +} + +// paymentFeatureSet defines a combination of optional payment features for +// testing feature interactions. +type paymentFeatureSet struct { + name string + mpp bool + amp bool + customRecords bool + blindedRoute bool + hopMetadata bool +} + +// TestMigrateLegacyPaymentWithNilHash tests migration of a legacy payment where +// the HTLC attempt's Hash field is nil. In older payments, the Hash wasn't +// stored on individual HTLCs; instead, the parent payment hash should be used. +func TestMigrateLegacyPaymentWithNilHash(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_legacy_nil_hash_payment")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket(paymentsIndexBucket) + if err != nil { + return err + } + + return createLegacyPaymentWithNilHash( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Verify the payment was migrated correctly. + var paymentID lntypes.Hash + copy(paymentID[:], paymentHash[:]) + + payment, err := sqlStore.FetchPayment(ctx, paymentID) + require.NoError(t, err) + require.NotNil(t, payment) + require.Len(t, payment.HTLCs, 1) + + // The HTLC should have the parent payment hash since its own Hash was + // nil. + require.NotNil(t, payment.HTLCs[0].Hash) + require.Equal(t, paymentID, *payment.HTLCs[0].Hash) +} + +// createLegacyPaymentWithNilHash creates a payment with an HTLC attempt that +// has a nil Hash field, simulating older payment data format. +func createLegacyPaymentWithNilHash(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists(hash[:]) + if err != nil { + return err + } + + var paymentID lntypes.Hash + copy(paymentID[:], hash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(50000), + CreationTime: time.Now().Add(-24 * time.Hour), + PaymentRequest: []byte("lnbc500n1legacy_payment"), + } + + var creationBuf bytes.Buffer + err = serializePaymentCreationInfo(&creationBuf, creationInfo) + if err != nil { + return err + } + err = paymentBucket.Put(paymentCreationInfoKey, creationBuf.Bytes()) + if err != nil { + return err + } + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, 100) + err = paymentBucket.Put(paymentSequenceKey, seqBytes) + if err != nil { + return err + } + + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + // Create HTLC attempt with nil Hash (legacy format). + sessionKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + hopKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + attemptInfo := &HTLCAttemptInfo{ + AttemptID: 1, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: 500000, + TotalAmount: 50000, + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex(hopKey.PubKey()), + ChannelID: 12345, + OutgoingTimeLock: 499500, + AmtToForward: 49000, + }, + }, + }, + AttemptTime: time.Now().Add(-2 * time.Hour), + Hash: nil, // Legacy: Hash is nil + } + + // Serialize the attempt info. Since Hash is nil, the serialization + // will not write the hash, simulating the legacy format. + attemptKey := make([]byte, len(htlcAttemptInfoKey)+8) + copy(attemptKey, htlcAttemptInfoKey) + byteOrder.PutUint64(attemptKey[len(htlcAttemptInfoKey):], 1) + + var attemptBuf bytes.Buffer + err = serializeHTLCAttemptInfo(&attemptBuf, attemptInfo) + if err != nil { + return err + } + err = htlcBucket.Put(attemptKey, attemptBuf.Bytes()) + if err != nil { + return err + } + + // Add settlement info. + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(hash), + SettleTime: time.Now().Add(-1 * time.Hour), + } + + settleKey := make([]byte, len(htlcSettleInfoKey)+8) + copy(settleKey, htlcSettleInfoKey) + byteOrder.PutUint64(settleKey[len(htlcSettleInfoKey):], 1) + + var settleBuf bytes.Buffer + err = serializeHTLCSettleInfo(&settleBuf, settleInfo) + if err != nil { + return err + } + err = htlcBucket.Put(settleKey, settleBuf.Bytes()) + if err != nil { + return err + } + + // Create index entry. + var idx bytes.Buffer + err = WriteElements(&idx, paymentIndexTypeHash, hash[:]) + if err != nil { + return err + } + + return indexBucket.Put(seqBytes, idx.Bytes()) +} + +// createPaymentWithFeatureSet creates a payment with a selected set of +// optional features for combination testing. +func createPaymentWithFeatureSet(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte, seqNum uint64, + features paymentFeatureSet, globalAttemptID *uint64) error { + t.Helper() + + if features.mpp && features.amp { + return fmt.Errorf("invalid feature set: mpp and amp") + } + + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists(hash[:]) + if err != nil { + return err + } + + var paymentID lntypes.Hash + copy(paymentID[:], hash[:]) + + creationTime := time.Date(2024, 7, 1, 12, 0, 0, 0, time.UTC). + Add(time.Duration(seqNum) * time.Minute) + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(100000), + CreationTime: creationTime, + PaymentRequest: []byte( + fmt.Sprintf("lnbc_test_%s", features.name), + ), + } + if features.customRecords { + creationInfo.FirstHopCustomRecords = lnwire.CustomRecords{ + 65560: []byte("combo_payment_custom_1"), + 65561: []byte("combo_payment_custom_2"), + } + } + + var b bytes.Buffer + err = serializePaymentCreationInfo(&b, creationInfo) + if err != nil { + return err + } + err = paymentBucket.Put(paymentCreationInfoKey, b.Bytes()) + if err != nil { + return err + } + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, seqNum) + err = paymentBucket.Put(paymentSequenceKey, seqBytes) + if err != nil { + return err + } + + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + sessionKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + var sourcePubKey Vertex + copy(sourcePubKey[:], sessionKey.PubKey().SerializeCompressed()) + + var sessionKeyBytes [32]byte + copy(sessionKeyBytes[:], sessionKey.Serialize()) + + baseAmt := lnwire.MilliSatoshi(100000) + hops := make([]*Hop, 3) + for i := 0; i < 3; i++ { + hopKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + amt := baseAmt - lnwire.MilliSatoshi(uint64(i)*100) + hop := &Hop{ + PubKeyBytes: NewVertex(hopKey.PubKey()), + ChannelID: uint64(700000 + i), + OutgoingTimeLock: uint32(700000 - i*40), + AmtToForward: amt, + } + if features.customRecords { + hop.CustomRecords = record.CustomSet{ + 65562 + uint64(i): []byte(fmt.Sprintf( + "combo_hop_%d", i, + )), + } + } + if features.hopMetadata { + hop.Metadata = []byte( + fmt.Sprintf("combo_metadata_%d", i), + ) + } + + if i == 2 { + if features.mpp { + var paymentAddr [32]byte + copy( + paymentAddr[:], + []byte("combo_mpp_payment_addr_1234"), + ) + hop.MPP = record.NewMPP( + baseAmt-200, paymentAddr, + ) + } + if features.amp { + var rootShare [32]byte + copy( + rootShare[:], + []byte("combo_amp_root_share_123456"), + ) + var setID [32]byte + copy( + setID[:], + []byte("combo_amp_set_id_12345678"), + ) + hop.AMP = record.NewAMP(rootShare, setID, 0) + } + if features.blindedRoute { + blindingKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + hop.BlindingPoint = blindingKey.PubKey() + hop.EncryptedData = []byte( + "combo_encrypted_blinded_data", + ) + hop.TotalAmtMsat = baseAmt - 200 + } + } + + hops[i] = hop + } + + routeInfo := Route{ + TotalTimeLock: 700000, + TotalAmount: baseAmt - 200, + SourcePubKey: sourcePubKey, + Hops: hops, + } + if features.customRecords { + routeInfo.FirstHopWireCustomRecords = lnwire.CustomRecords{ + 65565: []byte("combo_attempt_custom_1"), + 65566: []byte("combo_attempt_custom_2"), + } + } + + *globalAttemptID++ + attemptID := *globalAttemptID + attemptInfo := &HTLCAttemptInfo{ + AttemptID: attemptID, + sessionKey: sessionKeyBytes, + Route: routeInfo, + AttemptTime: creationTime.Add(time.Minute), + Hash: (*lntypes.Hash)(&hash), + } + + attemptKey := make([]byte, len(htlcAttemptInfoKey)+8) + copy(attemptKey, htlcAttemptInfoKey) + byteOrder.PutUint64(attemptKey[len(htlcAttemptInfoKey):], attemptID) + + var ab bytes.Buffer + err = serializeHTLCAttemptInfo(&ab, attemptInfo) + if err != nil { + return err + } + err = htlcBucket.Put(attemptKey, ab.Bytes()) + if err != nil { + return err + } + + settleInfo := &HTLCSettleInfo{ + Preimage: lntypes.Preimage(hash), + SettleTime: creationTime.Add(2 * time.Minute), + } + + settleKey := make([]byte, len(htlcSettleInfoKey)+8) + copy(settleKey, htlcSettleInfoKey) + byteOrder.PutUint64(settleKey[len(htlcSettleInfoKey):], attemptID) + + var sb bytes.Buffer + err = serializeHTLCSettleInfo(&sb, settleInfo) + if err != nil { + return err + } + err = htlcBucket.Put(settleKey, sb.Bytes()) + if err != nil { + return err + } + + var idx bytes.Buffer + err = WriteElements(&idx, paymentIndexTypeHash, hash[:]) + if err != nil { + return err + } + + return indexBucket.Put(seqBytes, idx.Bytes()) +} + +// TestMigrateInFlightPayment tests that a payment with an active (in-flight) +// HTLC attempt — i.e. a node that upgrades while a payment is still pending on +// the network — is migrated correctly. The HTLC attempt must appear in SQL +// without a settlement or failure resolution, and FetchInFlightPayments must +// return the payment after migration. +func TestMigrateInFlightPayment(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kvDB := setupTestKVDB(t) + + var paymentHash [32]byte + copy(paymentHash[:], []byte("test_inflight_payment_hash_1234")) + + err := kvdb.Update(kvDB, func(tx kvdb.RwTx) error { + paymentsBucket, err := tx.CreateTopLevelBucket( + paymentsRootBucket, + ) + if err != nil { + return err + } + + indexBucket, err := tx.CreateTopLevelBucket( + paymentsIndexBucket, + ) + if err != nil { + return err + } + + return createInFlightPayment( + t, paymentsBucket, indexBucket, paymentHash, + ) + }, func() {}) + require.NoError(t, err) + + sqlStore := setupTestSQLDB(t) + + err = runPaymentsMigration(ctx, kvDB, sqlStore) + require.NoError(t, err) + + // Verify the payment data matches between KV and SQL. + assertPaymentDataMatches(t, ctx, kvDB, sqlStore, paymentHash) + + // Additionally verify that FetchInFlightPayments returns this payment, + // confirming it is treated as in-flight by the SQL store. + inFlight, err := sqlStore.FetchInFlightPayments(ctx) + require.NoError(t, err) + require.Len(t, inFlight, 1) + + var paymentID lntypes.Hash + copy(paymentID[:], paymentHash[:]) + require.Equal(t, paymentID, inFlight[0].Info.PaymentIdentifier) + require.Equal(t, StatusInFlight, inFlight[0].Status) + require.Len(t, inFlight[0].HTLCs, 1) + require.Nil(t, inFlight[0].HTLCs[0].Settle) + require.Nil(t, inFlight[0].HTLCs[0].Failure) +} + +// createInFlightPayment creates a payment with an active (in-flight) HTLC +// attempt in the KV store. Unlike settled payments, no settle or fail +// resolution is written for the HTLC, simulating a payment that is still +// pending on the network at the time of migration. +func createInFlightPayment(t *testing.T, paymentsBucket, + indexBucket kvdb.RwBucket, hash [32]byte) error { + + t.Helper() + + paymentBucket, err := paymentsBucket.CreateBucketIfNotExists(hash[:]) + if err != nil { + return err + } + + var paymentID lntypes.Hash + copy(paymentID[:], hash[:]) + + creationInfo := &PaymentCreationInfo{ + PaymentIdentifier: paymentID, + Value: lnwire.MilliSatoshi(100000), + CreationTime: time.Now().Add(-1 * time.Hour), + PaymentRequest: []byte("lnbc1inflight_payment"), + } + + err = serializeAndPut( + paymentBucket, paymentCreationInfoKey, + func(w io.Writer) error { + return serializePaymentCreationInfo(w, creationInfo) + }, + ) + if err != nil { + return err + } + + seqBytes := make([]byte, 8) + byteOrder.PutUint64(seqBytes, 42) + if err := paymentBucket.Put(paymentSequenceKey, seqBytes); err != nil { + return err + } + + htlcBucket, err := paymentBucket.CreateBucketIfNotExists( + paymentHtlcsBucket, + ) + if err != nil { + return err + } + + _, sourcePubKey, sessionKeyBytes, err := generateSessionKey(t) + if err != nil { + return err + } + + hopKey, err := btcec.NewPrivateKey() + if err != nil { + return err + } + + const attemptID = 1 + attemptInfo := &HTLCAttemptInfo{ + AttemptID: attemptID, + sessionKey: sessionKeyBytes, + Route: Route{ + TotalTimeLock: 500000, + TotalAmount: 100000, + SourcePubKey: sourcePubKey, + Hops: []*Hop{ + { + PubKeyBytes: NewVertex( + hopKey.PubKey(), + ), + ChannelID: 12345, + OutgoingTimeLock: 499500, + AmtToForward: 99000, + }, + }, + }, + AttemptTime: time.Now().Add(-30 * time.Minute), + Hash: (*lntypes.Hash)(&hash), + } + + if err = writeHTLCAttempt(htlcBucket, attemptID, attemptInfo); err != nil { + return err + } + + return createIndexEntry(indexBucket, seqBytes, hash) +} diff --git a/payments/db/migration1/sql_store.go b/payments/db/migration1/sql_store.go new file mode 100644 index 000000000..2129656d9 --- /dev/null +++ b/payments/db/migration1/sql_store.go @@ -0,0 +1,2022 @@ +package migration1 + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "math" + "strconv" + "time" + + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/lnwire" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" +) + +// PaymentIntentType represents the type of payment intent. +type PaymentIntentType int16 + +const ( + // PaymentIntentTypeBolt11 indicates a BOLT11 invoice payment. + PaymentIntentTypeBolt11 PaymentIntentType = 0 +) + +// HTLCAttemptResolutionType represents the type of HTLC attempt resolution. +type HTLCAttemptResolutionType int32 + +const ( + // HTLCAttemptResolutionSettled indicates the HTLC attempt was settled + // successfully with a preimage. + HTLCAttemptResolutionSettled HTLCAttemptResolutionType = 1 + + // HTLCAttemptResolutionFailed indicates the HTLC attempt failed. + HTLCAttemptResolutionFailed HTLCAttemptResolutionType = 2 +) + +// SQLQueries is a subset of the sqlc.Querier interface that can be used to +// execute queries against the SQL payments tables. +// +//nolint:ll,interfacebloat +type SQLQueries interface { + /* + Payment DB read operations. + */ + FilterPayments(ctx context.Context, query sqlc.FilterPaymentsParams) ([]sqlc.FilterPaymentsRow, error) + FilterPaymentsDesc(ctx context.Context, query sqlc.FilterPaymentsDescParams) ([]sqlc.FilterPaymentsDescRow, error) + FetchPayment(ctx context.Context, paymentIdentifier []byte) (sqlc.FetchPaymentRow, error) + FetchPaymentsByIDs(ctx context.Context, paymentIDs []int64) ([]sqlc.FetchPaymentsByIDsRow, error) + + CountPayments(ctx context.Context) (int64, error) + + FetchHtlcAttemptsForPayments(ctx context.Context, paymentIDs []int64) ([]sqlc.FetchHtlcAttemptsForPaymentsRow, error) + FetchHtlcAttemptResolutionsForPayments(ctx context.Context, paymentIDs []int64) ([]sqlc.FetchHtlcAttemptResolutionsForPaymentsRow, error) + FetchAllInflightAttempts(ctx context.Context, arg sqlc.FetchAllInflightAttemptsParams) ([]sqlc.PaymentHtlcAttempt, error) + FetchHopsForAttempts(ctx context.Context, htlcAttemptIndices []int64) ([]sqlc.FetchHopsForAttemptsRow, error) + + FetchPaymentDuplicates(ctx context.Context, paymentID int64) ([]sqlc.PaymentDuplicate, error) + + FetchPaymentLevelFirstHopCustomRecords(ctx context.Context, paymentIDs []int64) ([]sqlc.PaymentFirstHopCustomRecord, error) + FetchRouteLevelFirstHopCustomRecords(ctx context.Context, htlcAttemptIndices []int64) ([]sqlc.PaymentAttemptFirstHopCustomRecord, error) + FetchHopLevelCustomRecords(ctx context.Context, hopIDs []int64) ([]sqlc.PaymentHopCustomRecord, error) + + /* + Payment DB write operations. + */ + InsertPaymentIntent(ctx context.Context, arg sqlc.InsertPaymentIntentParams) (int64, error) + InsertPayment(ctx context.Context, arg sqlc.InsertPaymentParams) (int64, error) + InsertPaymentFirstHopCustomRecord(ctx context.Context, arg sqlc.InsertPaymentFirstHopCustomRecordParams) error + + InsertHtlcAttempt(ctx context.Context, arg sqlc.InsertHtlcAttemptParams) (int64, error) + InsertRouteHop(ctx context.Context, arg sqlc.InsertRouteHopParams) (int64, error) + InsertRouteHopMpp(ctx context.Context, arg sqlc.InsertRouteHopMppParams) error + InsertRouteHopAmp(ctx context.Context, arg sqlc.InsertRouteHopAmpParams) error + InsertRouteHopBlinded(ctx context.Context, arg sqlc.InsertRouteHopBlindedParams) error + + InsertPaymentAttemptFirstHopCustomRecord(ctx context.Context, arg sqlc.InsertPaymentAttemptFirstHopCustomRecordParams) error + InsertPaymentHopCustomRecord(ctx context.Context, arg sqlc.InsertPaymentHopCustomRecordParams) error + + SettleAttempt(ctx context.Context, arg sqlc.SettleAttemptParams) error + FailAttempt(ctx context.Context, arg sqlc.FailAttemptParams) error + + FailPayment(ctx context.Context, arg sqlc.FailPaymentParams) (sql.Result, error) + + DeletePayment(ctx context.Context, paymentID int64) error + + // DeleteFailedAttempts removes all failed HTLCs from the db for a + // given payment. + DeleteFailedAttempts(ctx context.Context, paymentID int64) error +} + +// SQLMigrationQueries extends SQLQueries with the additional queries needed +// for the one-time migration from KV to SQL. Keeping them in a separate +// interface makes it clear which code paths are migration-only and prevents +// the regular store from accidentally depending on them. +// +//nolint:ll +type SQLMigrationQueries interface { + SQLQueries + + // FetchPaymentsByIDsMig is a migration-only batch fetch that returns + // payment data along with HTLC attempt counts for structural + // validation. + FetchPaymentsByIDsMig(ctx context.Context, paymentIDs []int64) ([]sqlc.FetchPaymentsByIDsMigRow, error) + + // InsertPaymentMig is a migration-only variant of InsertPayment that + // allows setting fail_reason when inserting historical payments, since + // for real payments they have not failed at creation time and so no + // failure reason would exist yet. + InsertPaymentMig(ctx context.Context, arg sqlc.InsertPaymentMigParams) (int64, error) + + // InsertPaymentDuplicateMig inserts a duplicate payment record during + // migration. + InsertPaymentDuplicateMig(ctx context.Context, arg sqlc.InsertPaymentDuplicateMigParams) (int64, error) +} + +// BatchedSQLQueries is a version of the SQLQueries that's capable +// of batched database operations. +type BatchedSQLQueries interface { + SQLQueries + sqldb.BatchedTx[SQLQueries] +} + +// SQLStore represents a storage backend. +type SQLStore struct { + cfg *SQLStoreConfig + db BatchedSQLQueries +} + +// A compile-time constraint to ensure SQLStore implements DB. +var _ DB = (*SQLStore)(nil) + +// SQLStoreConfig holds the configuration for the SQLStore. +type SQLStoreConfig struct { + // QueryConfig holds configuration values for SQL queries. + QueryCfg *sqldb.QueryConfig +} + +// NewSQLStore creates a new SQLStore instance given an open +// BatchedSQLPaymentsQueries storage backend. +func NewSQLStore(cfg *SQLStoreConfig, db BatchedSQLQueries, + options ...OptionModifier) (*SQLStore, error) { + + opts := DefaultOptions() + for _, applyOption := range options { + applyOption(opts) + } + + if opts.NoMigration { + return nil, fmt.Errorf("the NoMigration option is not yet " + + "supported for SQL stores") + } + + return &SQLStore{ + cfg: cfg, + db: db, + }, nil +} + +// fetchPaymentWithCompleteData fetches a payment with all its related data +// including attempts, hops, and custom records from the database. +// This is a convenience wrapper around the batch loading functions for single +// payment operations. +func fetchPaymentWithCompleteData(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, + dbPayment sqlc.PaymentAndIntent) (*MPPayment, error) { + + payment := dbPayment.GetPayment() + + // Load batch data for this single payment. + batchData, err := batchLoadPaymentDetailsData( + ctx, cfg, db, []int64{payment.ID}, + ) + if err != nil { + return nil, fmt.Errorf("failed to load batch data: %w", err) + } + + // Build the payment from the batch data. + return buildPaymentFromBatchData(dbPayment, batchData) +} + +// paymentsCompleteData holds the full payment data when batch loading base +// payment data and all the related data for a payment. +type paymentsCompleteData struct { + *paymentsBaseData + *paymentsDetailsData +} + +// batchLoadPayments loads the full payment data for a batch of payment IDs. +func batchLoadPayments(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, paymentIDs []int64) (*paymentsCompleteData, error) { + + baseData, err := batchLoadpaymentsBaseData(ctx, cfg, db, paymentIDs) + if err != nil { + return nil, fmt.Errorf("failed to load payment base data: %w", + err) + } + + batchData, err := batchLoadPaymentDetailsData(ctx, cfg, db, paymentIDs) + if err != nil { + return nil, fmt.Errorf("failed to load payment batch data: %w", + err) + } + + return &paymentsCompleteData{ + paymentsBaseData: baseData, + paymentsDetailsData: batchData, + }, nil +} + +// paymentsBaseData holds the base payment and intent data for a batch of +// payments. +type paymentsBaseData struct { + // paymentsAndIntents maps payment ID to its payment and intent data. + paymentsAndIntents map[int64]sqlc.PaymentAndIntent +} + +// batchLoadpaymentsBaseData loads the base payment and payment intent data for +// a batch of payment IDs. This complements loadPaymentsBatchData which loads +// related data (attempts, hops, custom records) but not the payment table +// and payment intent table data. +func batchLoadpaymentsBaseData(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, + paymentIDs []int64) (*paymentsBaseData, error) { + + baseData := &paymentsBaseData{ + paymentsAndIntents: make(map[int64]sqlc.PaymentAndIntent), + } + + if len(paymentIDs) == 0 { + return baseData, nil + } + + err := sqldb.ExecuteBatchQuery( + ctx, cfg, paymentIDs, + func(id int64) int64 { return id }, + func(ctx context.Context, ids []int64) ( + []sqlc.FetchPaymentsByIDsRow, error) { + + records, err := db.FetchPaymentsByIDs( + ctx, ids, + ) + + return records, err + }, + func(ctx context.Context, + payment sqlc.FetchPaymentsByIDsRow) error { + + baseData.paymentsAndIntents[payment.ID] = payment + + return nil + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch payment base "+ + "data: %w", err) + } + + return baseData, nil +} + +// paymentsRelatedData holds all the batch-loaded data for multiple payments. +// This does not include the base payment and intent data which is fetched +// separately. It includes the additional data like attempts, hops, hop custom +// records, and route custom records. +type paymentsDetailsData struct { + // paymentCustomRecords maps payment ID to its custom records. + paymentCustomRecords map[int64][]sqlc.PaymentFirstHopCustomRecord + + // attempts maps payment ID to its HTLC attempts. + attempts map[int64][]sqlc.FetchHtlcAttemptsForPaymentsRow + + // hopsByAttempt maps attempt index to its hops. + hopsByAttempt map[int64][]sqlc.FetchHopsForAttemptsRow + + // hopCustomRecords maps hop ID to its custom records. + hopCustomRecords map[int64][]sqlc.PaymentHopCustomRecord + + // routeCustomRecords maps attempt index to its route-level custom + // records. + routeCustomRecords map[int64][]sqlc.PaymentAttemptFirstHopCustomRecord +} + +// batchLoadPaymentCustomRecords loads payment-level custom records for a given +// set of payment IDs. It uses a batch query to fetch all custom records for +// the given payment IDs. +func batchLoadPaymentCustomRecords(ctx context.Context, + cfg *sqldb.QueryConfig, db SQLQueries, paymentIDs []int64, + batchData *paymentsDetailsData) error { + + return sqldb.ExecuteBatchQuery( + ctx, cfg, paymentIDs, + func(id int64) int64 { return id }, + func(ctx context.Context, ids []int64) ( + []sqlc.PaymentFirstHopCustomRecord, error) { + + //nolint:ll + records, err := db.FetchPaymentLevelFirstHopCustomRecords( + ctx, ids, + ) + + return records, err + }, + func(ctx context.Context, + record sqlc.PaymentFirstHopCustomRecord) error { + + paymentRecords := + batchData.paymentCustomRecords[record.PaymentID] + + batchData.paymentCustomRecords[record.PaymentID] = + append(paymentRecords, record) + + return nil + }, + ) +} + +// batchLoadHtlcAttempts loads HTLC attempts for all payments and returns all +// attempt indices. It uses a batch query to fetch all attempts for the given +// payment IDs. +func batchLoadHtlcAttempts(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, paymentIDs []int64, + batchData *paymentsDetailsData) ([]int64, error) { + + var allAttemptIndices []int64 + + err := sqldb.ExecuteBatchQuery( + ctx, cfg, paymentIDs, + func(id int64) int64 { return id }, + func(ctx context.Context, ids []int64) ( + []sqlc.FetchHtlcAttemptsForPaymentsRow, error) { + + return db.FetchHtlcAttemptsForPayments(ctx, ids) + }, + func(ctx context.Context, + attempt sqlc.FetchHtlcAttemptsForPaymentsRow) error { + + batchData.attempts[attempt.PaymentID] = append( + batchData.attempts[attempt.PaymentID], attempt, + ) + allAttemptIndices = append( + allAttemptIndices, attempt.AttemptIndex, + ) + + return nil + }, + ) + + return allAttemptIndices, err +} + +// batchLoadHopsForAttempts loads hops for all attempts and returns all hop IDs. +// It uses a batch query to fetch all hops for the given attempt indices. +func batchLoadHopsForAttempts(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, attemptIndices []int64, + batchData *paymentsDetailsData) ([]int64, error) { + + var hopIDs []int64 + + err := sqldb.ExecuteBatchQuery( + ctx, cfg, attemptIndices, + func(idx int64) int64 { return idx }, + func(ctx context.Context, indices []int64) ( + []sqlc.FetchHopsForAttemptsRow, error) { + + return db.FetchHopsForAttempts(ctx, indices) + }, + func(ctx context.Context, + hop sqlc.FetchHopsForAttemptsRow) error { + + attemptHops := + batchData.hopsByAttempt[hop.HtlcAttemptIndex] + + batchData.hopsByAttempt[hop.HtlcAttemptIndex] = + append(attemptHops, hop) + + hopIDs = append(hopIDs, hop.ID) + + return nil + }, + ) + + return hopIDs, err +} + +// batchLoadHopCustomRecords loads hop-level custom records for all hops. It +// uses a batch query to fetch all custom records for the given hop IDs. +func batchLoadHopCustomRecords(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, hopIDs []int64, batchData *paymentsDetailsData) error { + + return sqldb.ExecuteBatchQuery( + ctx, cfg, hopIDs, + func(id int64) int64 { return id }, + func(ctx context.Context, ids []int64) ( + []sqlc.PaymentHopCustomRecord, error) { + + return db.FetchHopLevelCustomRecords(ctx, ids) + }, + func(ctx context.Context, + record sqlc.PaymentHopCustomRecord) error { + + // TODO(ziggie): Can we get rid of this? + // This has to be in place otherwise the + // comparison will not match. + if record.Value == nil { + record.Value = []byte{} + } + + batchData.hopCustomRecords[record.HopID] = append( + batchData.hopCustomRecords[record.HopID], + record, + ) + + return nil + }, + ) +} + +// batchLoadRouteCustomRecords loads route-level first hop custom records for +// all attempts. It uses a batch query to fetch all custom records for the given +// attempt indices. +func batchLoadRouteCustomRecords(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, attemptIndices []int64, + batchData *paymentsDetailsData) error { + + return sqldb.ExecuteBatchQuery( + ctx, cfg, attemptIndices, + func(idx int64) int64 { return idx }, + func(ctx context.Context, indices []int64) ( + []sqlc.PaymentAttemptFirstHopCustomRecord, error) { + + return db.FetchRouteLevelFirstHopCustomRecords( + ctx, indices, + ) + }, + func(ctx context.Context, + record sqlc.PaymentAttemptFirstHopCustomRecord) error { + + idx := record.HtlcAttemptIndex + attemptRecords := batchData.routeCustomRecords[idx] + + batchData.routeCustomRecords[idx] = + append(attemptRecords, record) + + return nil + }, + ) +} + +// paymentStatusData holds lightweight resolution data for computing +// payment status efficiently during deletion operations. +type paymentStatusData struct { + // resolutionTypes maps payment ID to a list of resolution types + // for that payment's HTLC attempts. + resolutionTypes map[int64][]sql.NullInt32 +} + +// batchLoadPaymentResolutions loads only HTLC resolution types for multiple +// payments. This is a lightweight alternative to batchLoadPaymentsRelatedData +// that's optimized for operations that only need to determine payment status. +func batchLoadPaymentResolutions(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, paymentIDs []int64) (*paymentStatusData, error) { + + batchStatusData := &paymentStatusData{ + resolutionTypes: make(map[int64][]sql.NullInt32), + } + + if len(paymentIDs) == 0 { + return batchStatusData, nil + } + + // Use a batch query to fetch all resolution types for the given payment + // IDs. + err := sqldb.ExecuteBatchQuery( + ctx, cfg, paymentIDs, + func(id int64) int64 { return id }, + func(ctx context.Context, ids []int64) ( + []sqlc.FetchHtlcAttemptResolutionsForPaymentsRow, + error) { + + return db.FetchHtlcAttemptResolutionsForPayments( + ctx, ids, + ) + }, + //nolint:ll + func(ctx context.Context, + res sqlc.FetchHtlcAttemptResolutionsForPaymentsRow) error { + + // Group resolutions by payment ID. + batchStatusData.resolutionTypes[res.PaymentID] = append( + batchStatusData.resolutionTypes[res.PaymentID], + res.ResolutionType, + ) + + return nil + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch HTLC resolutions: %w", + err) + } + + return batchStatusData, nil +} + +// loadPaymentResolutions is a single-payment wrapper around +// batchLoadPaymentResolutions for convenience and to prevent duplicate queries +// so we reuse the same batch query for all payments. +func loadPaymentResolutions(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, paymentID int64) ([]sql.NullInt32, error) { + + batchData, err := batchLoadPaymentResolutions( + ctx, cfg, db, []int64{paymentID}, + ) + if err != nil { + return nil, err + } + + return batchData.resolutionTypes[paymentID], nil +} + +// computePaymentStatusFromResolutions determines the payment status from +// resolution types and failure reason without building the complete MPPayment +// structure. This is a lightweight version that builds minimal HTLCAttempt +// structures and delegates to decidePaymentStatus for consistency. +func computePaymentStatusFromResolutions(resolutionTypes []sql.NullInt32, + failReason sql.NullInt32) (PaymentStatus, error) { + + // Build minimal HTLCAttempt slice with only resolution info. + htlcs := make([]HTLCAttempt, len(resolutionTypes)) + for i, resType := range resolutionTypes { + if !resType.Valid { + // NULL resolution_type means in-flight (no Settle, no + // Failure). + continue + } + + switch HTLCAttemptResolutionType(resType.Int32) { + case HTLCAttemptResolutionSettled: + // Mark as settled (preimage details not needed for + // status). + htlcs[i].Settle = &HTLCSettleInfo{} + + case HTLCAttemptResolutionFailed: + // Mark as failed (failure details not needed for + // status). + htlcs[i].Failure = &HTLCFailInfo{} + + default: + return 0, fmt.Errorf("unknown resolution type: %v", + resType.Int32) + } + } + + // Convert fail reason to FailureReason pointer. + var failureReason *FailureReason + if failReason.Valid { + reason := FailureReason(failReason.Int32) + failureReason = &reason + } + + // Use the existing status decision logic. + return decidePaymentStatus(htlcs, failureReason) +} + +// batchLoadPaymentDetailsData loads all related data for multiple payments in +// batch. It uses a batch queries to fetch all data for the given payment IDs. +func batchLoadPaymentDetailsData(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, paymentIDs []int64) (*paymentsDetailsData, error) { + + batchData := &paymentsDetailsData{ + paymentCustomRecords: make( + map[int64][]sqlc.PaymentFirstHopCustomRecord, + ), + attempts: make( + map[int64][]sqlc.FetchHtlcAttemptsForPaymentsRow, + ), + hopsByAttempt: make( + map[int64][]sqlc.FetchHopsForAttemptsRow, + ), + hopCustomRecords: make( + map[int64][]sqlc.PaymentHopCustomRecord, + ), + routeCustomRecords: make( + map[int64][]sqlc.PaymentAttemptFirstHopCustomRecord, + ), + } + + if len(paymentIDs) == 0 { + return batchData, nil + } + + // Load payment-level custom records. + err := batchLoadPaymentCustomRecords( + ctx, cfg, db, paymentIDs, batchData, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch payment custom "+ + "records: %w", err) + } + + // Load HTLC attempts and collect attempt indices. + allAttemptIndices, err := batchLoadHtlcAttempts( + ctx, cfg, db, paymentIDs, batchData, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch HTLC attempts: %w", + err) + } + + if len(allAttemptIndices) == 0 { + // No attempts, return early. + return batchData, nil + } + + // Load hops for all attempts and collect hop IDs. + hopIDs, err := batchLoadHopsForAttempts( + ctx, cfg, db, allAttemptIndices, batchData, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch hops for attempts: %w", + err) + } + + // Load hop-level custom records if there are any hops. + if len(hopIDs) > 0 { + err = batchLoadHopCustomRecords(ctx, cfg, db, hopIDs, batchData) + if err != nil { + return nil, fmt.Errorf("failed to fetch hop custom "+ + "records: %w", err) + } + } + + // Load route-level first hop custom records. + err = batchLoadRouteCustomRecords( + ctx, cfg, db, allAttemptIndices, batchData, + ) + if err != nil { + return nil, fmt.Errorf("failed to fetch route custom "+ + "records: %w", err) + } + + return batchData, nil +} + +// buildPaymentFromBatchData builds a complete MPPayment from a database payment +// and pre-loaded batch data. +func buildPaymentFromBatchData(dbPayment sqlc.PaymentAndIntent, + batchData *paymentsDetailsData) (*MPPayment, error) { + + // The query will only return BOLT 11 payment intents or intents with + // no intent type set. + paymentIntent := dbPayment.GetPaymentIntent() + paymentRequest := paymentIntent.IntentPayload + + payment := dbPayment.GetPayment() + + // Get payment-level custom records from batch data. + customRecords := batchData.paymentCustomRecords[payment.ID] + + // Convert to the FirstHopCustomRecords map. + var firstHopCustomRecords lnwire.CustomRecords + if len(customRecords) > 0 { + firstHopCustomRecords = make(lnwire.CustomRecords) + for _, record := range customRecords { + firstHopCustomRecords[uint64(record.Key)] = record.Value + } + } + + // Convert database payment data to the PaymentCreationInfo struct. + info := dbPaymentToCreationInfo( + payment.PaymentIdentifier, payment.AmountMsat, + payment.CreatedAt, paymentRequest, firstHopCustomRecords, + ) + + // Get all HTLC attempts from batch data for a given payment. + dbAttempts := batchData.attempts[payment.ID] + + // Convert all attempts to HTLCAttempt structs using the pre-loaded + // batch data. + attempts := make([]HTLCAttempt, 0, len(dbAttempts)) + for _, dbAttempt := range dbAttempts { + attemptIndex := dbAttempt.AttemptIndex + // Convert the batch row type to the single row type. + attempt, err := dbAttemptToHTLCAttempt( + dbAttempt, batchData.hopsByAttempt[attemptIndex], + batchData.hopCustomRecords, + batchData.routeCustomRecords[attemptIndex], + ) + if err != nil { + return nil, fmt.Errorf("failed to convert attempt "+ + "%d: %w", attemptIndex, err) + } + attempts = append(attempts, *attempt) + } + + // Set the failure reason if present. + // + // TODO(ziggie): Rename it to Payment Memo in the database? + var failureReason *FailureReason + if payment.FailReason.Valid { + reason := FailureReason(payment.FailReason.Int32) + failureReason = &reason + } + + mpPayment := &MPPayment{ + SequenceNum: uint64(payment.ID), + Info: info, + HTLCs: attempts, + FailureReason: failureReason, + } + + // The status and state will be determined by calling + // SetState after construction. + if err := mpPayment.SetState(); err != nil { + return nil, fmt.Errorf("failed to set payment state: %w", err) + } + + return mpPayment, nil +} + +// QueryPayments queries and retrieves payments from the database with support +// for filtering, pagination, and efficient batch loading of related data. +// +// The function accepts a Query parameter that controls: +// - Pagination: IndexOffset specifies where to start (exclusive), and +// MaxPayments limits the number of results returned +// - Ordering: Reversed flag determines if results are returned in reverse +// chronological order +// - Filtering: CreationDateStart/End filter by creation time, and +// IncludeIncomplete controls whether non-succeeded payments are included +// - Metadata: CountTotal flag determines if the total payment count should +// be calculated +// +// The function optimizes performance by loading all related data (HTLCs, +// sequences, failure reasons, etc.) for multiple payments in a single batch +// query, rather than fetching each payment's data individually. +// +// Returns a Response containing: +// - Payments: the list of matching payments with complete data +// - FirstIndexOffset/LastIndexOffset: pagination cursors for the first and +// last payment in the result set +// - TotalCount: total number of payments in the database (if CountTotal was +// requested, otherwise 0) +// +// This is part of the DB interface. +func (s *SQLStore) QueryPayments(ctx context.Context, query Query) (Response, + error) { + + if query.MaxPayments == 0 { + return Response{}, fmt.Errorf("max payments must be non-zero") + } + + var ( + allPayments []*MPPayment + totalCount int64 + initialCursor int64 + ) + + extractCursor := func(row sqlc.FilterPaymentsRow) int64 { + return row.Payment.ID + } + + err := s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + // We first count all payments to determine the total count + // if requested. + if query.CountTotal { + totalPayments, err := db.CountPayments(ctx) + if err != nil { + return fmt.Errorf("failed to count "+ + "payments: %w", err) + } + totalCount = totalPayments + } + + // collectFunc extracts the payment ID from each payment row. + collectFunc := func(row sqlc.FilterPaymentsRow) (int64, error) { + return row.Payment.ID, nil + } + + // batchDataFunc loads all related data for a batch of payments. + batchDataFunc := func(ctx context.Context, paymentIDs []int64) ( + *paymentsDetailsData, error) { + + return batchLoadPaymentDetailsData( + ctx, s.cfg.QueryCfg, db, paymentIDs, + ) + } + + // processPayment processes each payment with the batch-loaded + // data. + processPayment := func(ctx context.Context, + dbPayment sqlc.FilterPaymentsRow, + batchData *paymentsDetailsData) error { + + // Build the payment from the pre-loaded batch data. + mpPayment, err := buildPaymentFromBatchData( + dbPayment, batchData, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment "+ + "with complete data: %w", err) + } + + // To keep compatibility with the old API, we only + // return non-succeeded payments if requested. + if mpPayment.Status != StatusSucceeded && + !query.IncludeIncomplete { + + return nil + } + + if uint64(len(allPayments)) >= query.MaxPayments { + return errMaxPaymentsReached + } + + allPayments = append(allPayments, mpPayment) + + return nil + } + + //nolint:ll + convertFilterPaymentsDescRows := func( + rows []sqlc.FilterPaymentsDescRow) []sqlc.FilterPaymentsRow { + + out := make([]sqlc.FilterPaymentsRow, len(rows)) + for i, row := range rows { + out[i] = sqlc.FilterPaymentsRow{ + Payment: row.Payment, + IntentType: row.IntentType, + IntentPayload: row.IntentPayload, + } + } + + return out + } + + queryFunc := func(ctx context.Context, lastID int64, + limit int32) ([]sqlc.FilterPaymentsRow, error) { + + // Default date bounds: epoch start and far + // future. These are always provided so the SQL + // query uses simple comparisons instead of + // COALESCE (which causes type mismatch on + // Postgres) or OR-based optional filters (which + // can prevent index usage). + createdAfter := time.Unix(0, 0).UTC() + if query.CreationDateStart != 0 { + createdAfter = time.Unix( + query.CreationDateStart, 0, + ).UTC() + } + + createdBefore := time.Date( + 9999, 12, 31, 23, 59, 59, 0, time.UTC, + ) + if query.CreationDateEnd != 0 { + createdBefore = time.Unix( + query.CreationDateEnd, 0, + ).UTC() + } + + filterParams := sqlc.FilterPaymentsParams{ + NumLimit: limit, + CreatedAfter: createdAfter, + CreatedBefore: createdBefore, + // For now there only BOLT 11 payment intents + // exist. + IntentType: sqldb.SQLInt16( + PaymentIntentTypeBolt11, + ), + } + + if query.Reversed { + filterParams.IndexOffsetLet = sqldb.SQLInt64( + lastID, + ) + } else { + filterParams.IndexOffsetGet = sqldb.SQLInt64( + lastID, + ) + } + + if query.Reversed { + rows, err := db.FilterPaymentsDesc( + ctx, sqlc.FilterPaymentsDescParams( + filterParams, + ), + ) + if err != nil { + return nil, err + } + + return convertFilterPaymentsDescRows(rows), nil + } + + return db.FilterPayments(ctx, filterParams) + } + + if query.Reversed { + if query.IndexOffset == 0 { + initialCursor = int64(math.MaxInt64) + } else { + initialCursor = int64(query.IndexOffset) + } + } else { + initialCursor = int64(query.IndexOffset) + } + + return sqldb.ExecuteCollectAndBatchWithSharedDataQuery( + ctx, s.cfg.QueryCfg, initialCursor, queryFunc, + extractCursor, collectFunc, batchDataFunc, + processPayment, + ) + }, func() { + allPayments = nil + }) + + // We make sure we don't return an error if we reached the maximum + // number of payments. Which is the pagination limit for the query + // itself. + if err != nil && !errors.Is(err, errMaxPaymentsReached) { + return Response{}, fmt.Errorf("failed to query payments: %w", + err) + } + + // Handle case where no payments were found + if len(allPayments) == 0 { + return Response{ + Payments: allPayments, + FirstIndexOffset: 0, + LastIndexOffset: 0, + TotalCount: uint64(totalCount), + }, nil + } + + // If the query was reversed, we need to reverse the payment list + // to match the kvstore behavior and return payments in forward order. + if query.Reversed { + for i, j := 0, len(allPayments)-1; i < j; i, j = i+1, j-1 { + allPayments[i], allPayments[j] = allPayments[j], + allPayments[i] + } + } + + return Response{ + Payments: allPayments, + FirstIndexOffset: allPayments[0].SequenceNum, + LastIndexOffset: allPayments[len(allPayments)-1].SequenceNum, + TotalCount: uint64(totalCount), + }, nil +} + +// fetchPaymentByHash fetches a payment by its hash from the database. It is a +// convenience wrapper around the FetchPayment method and checks for +// no rows error and returns ErrPaymentNotInitiated if no payment is found. +func fetchPaymentByHash(ctx context.Context, db SQLQueries, + paymentHash lntypes.Hash) (sqlc.FetchPaymentRow, error) { + + dbPayment, err := db.FetchPayment(ctx, paymentHash[:]) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return dbPayment, fmt.Errorf("failed to fetch payment: %w", err) + } + + if errors.Is(err, sql.ErrNoRows) { + return dbPayment, ErrPaymentNotInitiated + } + + return dbPayment, nil +} + +// FetchPayment retrieves a complete payment record from the database by its +// payment hash. The returned MPPayment includes all payment metadata such as +// creation info, payment status, current state, all HTLC attempts (both +// successful and failed), and the failure reason if the payment has been +// marked as failed. +// +// Returns ErrPaymentNotInitiated if no payment with the given hash exists. +// +// This is part of the DB interface. +func (s *SQLStore) FetchPayment(ctx context.Context, + paymentHash lntypes.Hash) (*MPPayment, error) { + + var mpPayment *MPPayment + + err := s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + dbPayment, err := fetchPaymentByHash(ctx, db, paymentHash) + if err != nil { + return err + } + + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + return nil + }, sqldb.NoOpReset) + if err != nil { + return nil, err + } + + return mpPayment, nil +} + +// FetchInFlightPayments retrieves all payments that have HTLC attempts +// currently in flight (not yet settled or failed). These are payments with at +// least one HTLC attempt that has been registered but has no resolution record. +// +// The SQLStore implementation provides a significant performance improvement +// over the KVStore implementation by using targeted SQL queries instead of +// scanning all payments. +// +// This method is part of the PaymentReader interface, which is embedded in the +// DB interface. It's typically called during node startup to resume monitoring +// of pending payments and ensure HTLCs are properly tracked. +// +// TODO(ziggie): Consider changing the interface to use a callback or iterator +// pattern instead of returning all payments at once. This would allow +// processing payments one at a time without holding them all in memory +// simultaneously: +// - Callback: func FetchInFlightPayments(ctx, func(*MPPayment) error) error +// - Iterator: func FetchInFlightPayments(ctx) (PaymentIterator, error) +// +// While inflight payments are typically a small subset, this would improve +// memory efficiency for nodes with unusually high numbers of concurrent +// payments and would better leverage the existing pagination infrastructure. +func (s *SQLStore) FetchInFlightPayments(ctx context.Context) ([]*MPPayment, + error) { + + var mpPayments []*MPPayment + + err := s.db.ExecTx(ctx, sqldb.ReadTxOpt(), func(db SQLQueries) error { + // Track which payment IDs we've already processed across all + // pages to avoid loading the same payment multiple times when + // multiple inflight attempts belong to the same payment. + processedPayments := make(map[int64]*MPPayment) + + extractCursor := func(row sqlc.PaymentHtlcAttempt) int64 { + return row.AttemptIndex + } + + // collectFunc extracts the payment ID from each attempt row. + collectFunc := func(row sqlc.PaymentHtlcAttempt) ( + int64, error) { + + return row.PaymentID, nil + } + + // batchDataFunc loads payment data for a batch of payment IDs, + // but only for IDs we haven't processed yet. + batchDataFunc := func(ctx context.Context, + paymentIDs []int64) (*paymentsCompleteData, error) { + + // Filter out already-processed payment IDs. + uniqueIDs := make([]int64, 0, len(paymentIDs)) + for _, id := range paymentIDs { + _, processed := processedPayments[id] + if !processed { + uniqueIDs = append(uniqueIDs, id) + } + } + + // If uniqueIDs is empty, the batch load will return + // empty batch data. + return batchLoadPayments( + ctx, s.cfg.QueryCfg, db, uniqueIDs, + ) + } + + // processAttempt processes each attempt. We only build and + // store the payment once per unique payment ID. + processAttempt := func(ctx context.Context, + row sqlc.PaymentHtlcAttempt, + batchData *paymentsCompleteData) error { + + // Skip if we've already processed this payment. + _, processed := processedPayments[row.PaymentID] + if processed { + return nil + } + + dbPayment := batchData.paymentsAndIntents[row.PaymentID] + + // Build the payment from batch data. + mpPayment, err := buildPaymentFromBatchData( + dbPayment, batchData.paymentsDetailsData, + ) + if err != nil { + return fmt.Errorf("failed to build payment: %w", + err) + } + + // Store in our processed map. + processedPayments[row.PaymentID] = mpPayment + + return nil + } + + queryFunc := func(ctx context.Context, lastAttemptIndex int64, + limit int32) ([]sqlc.PaymentHtlcAttempt, + error) { + + return db.FetchAllInflightAttempts(ctx, + sqlc.FetchAllInflightAttemptsParams{ + AttemptIndex: lastAttemptIndex, + Limit: limit, + }, + ) + } + + err := sqldb.ExecuteCollectAndBatchWithSharedDataQuery( + ctx, s.cfg.QueryCfg, int64(-1), queryFunc, + extractCursor, collectFunc, batchDataFunc, + processAttempt, + ) + if err != nil { + return err + } + + // Convert map to slice. + mpPayments = make([]*MPPayment, 0, len(processedPayments)) + for _, payment := range processedPayments { + mpPayments = append(mpPayments, payment) + } + + return nil + }, func() { + mpPayments = nil + }) + if err != nil { + return nil, fmt.Errorf("failed to fetch inflight "+ + "payments: %w", err) + } + + return mpPayments, nil +} + +// DeleteFailedAttempts removes all failed HTLC attempts from the database for +// the specified payment, while preserving the payment record itself and any +// successful or in-flight attempts. +// +// The method performs the following validations before deletion: +// - StatusInitiated: Can delete failed attempts +// - StatusInFlight: Cannot delete, returns ErrPaymentInFlight (active HTLCs +// still on the network) +// - StatusSucceeded: Can delete failed attempts (payment completed) +// - StatusFailed: Can delete failed attempts (payment permanently failed) +// +// This method is idempotent - calling it multiple times on the same payment +// has no adverse effects. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface. It represents +// the final step (step 5) in the payment lifecycle control flow and should be +// called after a payment reaches a terminal state (succeeded or permanently +// failed) to clean up historical failed attempts. +func (s *SQLStore) DeleteFailedAttempts(ctx context.Context, + paymentHash lntypes.Hash) error { + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + dbPayment, err := fetchPaymentByHash(ctx, db, paymentHash) + if err != nil { + return err + } + + paymentStatus, err := computePaymentStatusFromDB( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + if err := paymentStatus.removable(); err != nil { + return fmt.Errorf("cannot delete failed "+ + "attempts for payment %v: %w", paymentHash, err) + } + + // Then we delete the failed attempts for this payment. + return db.DeleteFailedAttempts(ctx, dbPayment.GetPayment().ID) + }, sqldb.NoOpReset) + if err != nil { + return fmt.Errorf("failed to delete failed attempts for "+ + "payment %v: %w", paymentHash, err) + } + + return nil +} + +// computePaymentStatusFromDB computes the payment status by fetching minimal +// data from the database. This is a lightweight query optimized for SQL that +// doesn't load route data, making it significantly more efficient than +// FetchPayment when only the status is needed. +func computePaymentStatusFromDB(ctx context.Context, cfg *sqldb.QueryConfig, + db SQLQueries, dbPayment sqlc.PaymentAndIntent) (PaymentStatus, error) { + + payment := dbPayment.GetPayment() + + // Load the resolution types for the payment. + resolutionTypes, err := loadPaymentResolutions( + ctx, cfg, db, payment.ID, + ) + if err != nil { + return 0, fmt.Errorf("failed to load payment resolutions: %w", + err) + } + + // Use the lightweight status computation. + status, err := computePaymentStatusFromResolutions( + resolutionTypes, payment.FailReason, + ) + if err != nil { + return 0, fmt.Errorf("failed to compute payment status: %w", + err) + } + + return status, nil +} + +// DeletePayment removes a payment or its failed HTLC attempts from the +// database based on the failedAttemptsOnly flag. +// +// If failedAttemptsOnly is true, this method deletes only the failed HTLC +// attempts for the payment while preserving the payment record itself and any +// successful or in-flight attempts. This is useful for cleaning up historical +// failed attempts after a payment reaches a terminal state. +// +// If failedAttemptsOnly is false, this method deletes the entire payment +// record including all payment metadata, payment creation info, all HTLC +// attempts (both failed and successful), and associated data such as payment +// intents and custom records. +// +// Before deletion, this method validates the payment status to ensure it's +// safe to delete: +// - StatusInitiated: Can be deleted (no HTLCs sent yet) +// - StatusInFlight: Cannot be deleted, returns ErrPaymentInFlight (active +// HTLCs on the network) +// - StatusSucceeded: Can be deleted (payment completed successfully) +// - StatusFailed: Can be deleted (payment has failed permanently) +// +// Returns an error if the payment has in-flight HTLCs or if the payment +// doesn't exist. +// +// This method is part of the PaymentWriter interface, which is embedded in +// the DB interface. +func (s *SQLStore) DeletePayment(ctx context.Context, paymentHash lntypes.Hash, + failedHtlcsOnly bool) error { + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + dbPayment, err := fetchPaymentByHash(ctx, db, paymentHash) + if err != nil { + return err + } + + paymentStatus, err := computePaymentStatusFromDB( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + if err := paymentStatus.removable(); err != nil { + return fmt.Errorf("payment %v cannot be deleted: %w", + paymentHash, err) + } + + // If we are only deleting failed HTLCs, we delete them. + if failedHtlcsOnly { + return db.DeleteFailedAttempts( + ctx, dbPayment.GetPayment().ID, + ) + } + + // In case we are not deleting failed HTLCs, we delete the + // payment which will cascade delete all related data. + return db.DeletePayment(ctx, dbPayment.GetPayment().ID) + }, sqldb.NoOpReset) + if err != nil { + return fmt.Errorf("failed to delete failed attempts for "+ + "payment %v: %w", paymentHash, err) + } + + return nil +} + +// InitPayment creates a new payment record in the database with the given +// payment hash and creation info. +// +// Before creating the payment, this method checks if a payment with the same +// hash already exists and validates whether initialization is allowed based on +// the existing payment's status: +// - StatusInitiated: Returns ErrPaymentExists (payment already created, +// HTLCs may be in flight) +// - StatusInFlight: Returns ErrPaymentInFlight (payment currently being +// attempted) +// - StatusSucceeded: Returns ErrAlreadyPaid (payment already succeeded) +// - StatusFailed: Allows retry by deleting the old payment record and +// creating a new one +// +// If no existing payment is found, a new payment record is created with +// StatusInitiated and stored with all associated metadata. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface, representing +// the first step in the payment lifecycle control flow. +func (s *SQLStore) InitPayment(ctx context.Context, paymentHash lntypes.Hash, + paymentCreationInfo *PaymentCreationInfo) error { + + // Create the payment in the database. + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + existingPayment, err := db.FetchPayment(ctx, paymentHash[:]) + switch { + // A payment with this hash already exists. We need to check its + // status to see if we can re-initialize. + case err == nil: + paymentStatus, err := computePaymentStatusFromDB( + ctx, s.cfg.QueryCfg, db, existingPayment, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + // Check if the payment is initializable otherwise + // we'll return early. + if err := paymentStatus.initializable(); err != nil { + return fmt.Errorf("payment is not "+ + "initializable: %w", err) + } + + // If the initializable check above passes, then the + // existing payment has failed. So we delete it and + // all of its previous artifacts. We rely on + // cascading deletes to clean up the rest. + err = db.DeletePayment(ctx, existingPayment.Payment.ID) + if err != nil { + return fmt.Errorf("failed to delete "+ + "payment: %w", err) + } + + // An unexpected error occurred while fetching the payment. + case !errors.Is(err, sql.ErrNoRows): + // Some other error occurred + return fmt.Errorf("failed to check existing "+ + "payment: %w", err) + + // The payment does not yet exist, so we can proceed. + default: + } + + // Insert the payment first to get its ID. + paymentID, err := db.InsertPayment( + ctx, sqlc.InsertPaymentParams{ + AmountMsat: int64( + paymentCreationInfo.Value, + ), + CreatedAt: paymentCreationInfo. + CreationTime.UTC(), + PaymentIdentifier: paymentHash[:], + }, + ) + if err != nil { + return fmt.Errorf("failed to insert payment: %w", err) + } + + // If there's a payment request, insert the payment intent. + if len(paymentCreationInfo.PaymentRequest) > 0 { + _, err = db.InsertPaymentIntent( + ctx, sqlc.InsertPaymentIntentParams{ + PaymentID: paymentID, + IntentType: int16( + PaymentIntentTypeBolt11, + ), + IntentPayload: paymentCreationInfo. + PaymentRequest, + }, + ) + if err != nil { + return fmt.Errorf("failed to insert "+ + "payment intent: %w", err) + } + } + + firstHopCustomRecords := paymentCreationInfo. + FirstHopCustomRecords + + for key, value := range firstHopCustomRecords { + err = db.InsertPaymentFirstHopCustomRecord( + ctx, + sqlc.InsertPaymentFirstHopCustomRecordParams{ + PaymentID: paymentID, + Key: int64(key), + Value: value, + }, + ) + if err != nil { + return fmt.Errorf("failed to insert "+ + "payment first hop custom "+ + "record: %w", err) + } + } + + return nil + }, sqldb.NoOpReset) + if err != nil { + return fmt.Errorf("failed to initialize payment: %w", err) + } + + return nil +} + +// insertRouteHops inserts all route hop data for a given set of hops. +func (s *SQLStore) insertRouteHops(ctx context.Context, db SQLQueries, + hops []*Hop, attemptID uint64) error { + + for i, hop := range hops { + // Insert the basic route hop data and get the generated ID. + hopID, err := db.InsertRouteHop(ctx, sqlc.InsertRouteHopParams{ + HtlcAttemptIndex: int64(attemptID), + HopIndex: int32(i), + PubKey: hop.PubKeyBytes[:], + Scid: strconv.FormatUint( + hop.ChannelID, 10, + ), + OutgoingTimeLock: int32(hop.OutgoingTimeLock), + AmtToForward: int64(hop.AmtToForward), + MetaData: hop.Metadata, + }) + if err != nil { + return fmt.Errorf("failed to insert route hop: %w", err) + } + + // Insert the per-hop custom records. + if len(hop.CustomRecords) > 0 { + for key, value := range hop.CustomRecords { + err = db.InsertPaymentHopCustomRecord( + ctx, + sqlc.InsertPaymentHopCustomRecordParams{ + HopID: hopID, + Key: int64(key), + Value: value, + }) + if err != nil { + return fmt.Errorf("failed to insert "+ + "payment hop custom record: %w", + err) + } + } + } + + // Insert MPP data if present. + if hop.MPP != nil { + paymentAddr := hop.MPP.PaymentAddr() + err = db.InsertRouteHopMpp( + ctx, sqlc.InsertRouteHopMppParams{ + HopID: hopID, + PaymentAddr: paymentAddr[:], + TotalMsat: int64(hop.MPP.TotalMsat()), + }) + if err != nil { + return fmt.Errorf("failed to insert "+ + "route hop MPP: %w", err) + } + } + + // Insert AMP data if present. + if hop.AMP != nil { + rootShare := hop.AMP.RootShare() + setID := hop.AMP.SetID() + err = db.InsertRouteHopAmp( + ctx, sqlc.InsertRouteHopAmpParams{ + HopID: hopID, + RootShare: rootShare[:], + SetID: setID[:], + ChildIndex: int32(hop.AMP.ChildIndex()), + }) + if err != nil { + return fmt.Errorf("failed to insert "+ + "route hop AMP: %w", err) + } + } + + // Insert blinded route data if present. Every hop in the + // blinded path must have an encrypted data record. If the + // encrypted data is not present, we skip the insertion. + if hop.EncryptedData == nil { + continue + } + + // The introduction point has a blinding point set. + var blindingPointBytes []byte + if hop.BlindingPoint != nil { + blindingPointBytes = hop.BlindingPoint. + SerializeCompressed() + } + + // The total amount is only set for the final hop in a + // blinded path. + totalAmtMsat := sql.NullInt64{} + if i == len(hops)-1 { + totalAmtMsat = sql.NullInt64{ + Int64: int64(hop.TotalAmtMsat), + Valid: true, + } + } + + err = db.InsertRouteHopBlinded(ctx, + sqlc.InsertRouteHopBlindedParams{ + HopID: hopID, + EncryptedData: hop.EncryptedData, + BlindingPoint: blindingPointBytes, + BlindedPathTotalAmt: totalAmtMsat, + }, + ) + if err != nil { + return fmt.Errorf("failed to insert "+ + "route hop blinded: %w", err) + } + } + + return nil +} + +// RegisterAttempt atomically records a new HTLC attempt for the specified +// payment. The attempt includes the attempt ID, session key, route information +// (hops, timelocks, amounts), and optional data such as MPP/AMP parameters, +// blinded route data, and custom records. +// +// Returns the updated MPPayment with the new attempt appended to the HTLCs +// slice, and the payment state recalculated. Returns an error if the payment +// doesn't exist or validation fails. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface. It represents +// step 2 in the payment lifecycle control flow, called after InitPayment and +// potentially multiple times for multi-path payments. +func (s *SQLStore) RegisterAttempt(ctx context.Context, + paymentHash lntypes.Hash, attempt *HTLCAttemptInfo) (*MPPayment, + error) { + + var mpPayment *MPPayment + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + // Make sure the payment exists. + dbPayment, err := db.FetchPayment(ctx, paymentHash[:]) + if err != nil { + return err + } + + // We fetch the complete payment to determine if the payment is + // registrable. + // + // TODO(ziggie): We could improve the query here since only + // the last hop data is needed here not the complete payment + // data. + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + if err := mpPayment.Registrable(); err != nil { + return fmt.Errorf("htlc attempt not registrable: %w", + err) + } + + // Verify the attempt is compatible with the existing payment. + if err := verifyAttempt(mpPayment, attempt); err != nil { + return fmt.Errorf("failed to verify attempt: %w", err) + } + + // Register the plain HTLC attempt next. + sessionKey := attempt.SessionKey() + sessionKeyBytes := sessionKey.Serialize() + + _, err = db.InsertHtlcAttempt(ctx, sqlc.InsertHtlcAttemptParams{ + PaymentID: dbPayment.Payment.ID, + AttemptIndex: int64(attempt.AttemptID), + SessionKey: sessionKeyBytes, + AttemptTime: attempt.AttemptTime, + PaymentHash: paymentHash[:], + FirstHopAmountMsat: int64( + attempt.Route.FirstHopAmount.Val.Int(), + ), + RouteTotalTimeLock: int32(attempt.Route.TotalTimeLock), + RouteTotalAmount: int64(attempt.Route.TotalAmount), + RouteSourceKey: attempt.Route.SourcePubKey[:], + }) + if err != nil { + return fmt.Errorf("failed to insert HTLC "+ + "attempt: %w", err) + } + + // Insert the route level first hop custom records. + attemptFirstHopCustomRecords := attempt.Route. + FirstHopWireCustomRecords + + for key, value := range attemptFirstHopCustomRecords { + //nolint:ll + err = db.InsertPaymentAttemptFirstHopCustomRecord( + ctx, + sqlc.InsertPaymentAttemptFirstHopCustomRecordParams{ + HtlcAttemptIndex: int64(attempt.AttemptID), + Key: int64(key), + Value: value, + }, + ) + if err != nil { + return fmt.Errorf("failed to insert "+ + "payment attempt first hop custom "+ + "record: %w", err) + } + } + + // Insert the route hops. + err = s.insertRouteHops( + ctx, db, attempt.Route.Hops, attempt.AttemptID, + ) + if err != nil { + return fmt.Errorf("failed to insert route hops: %w", + err) + } + + // We fetch the HTLC attempts again to recalculate the payment + // state after the attempt is registered. This also makes sure + // we have the right data in case multiple attempts are + // registered concurrently. + // + // NOTE: While the caller is responsible for serializing calls + // to RegisterAttempt per payment hash (see PaymentControl + // interface), we still refetch here to guarantee we return + // consistent, up-to-date data that reflects all changes made + // within this transaction. + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + return nil + }, func() { + mpPayment = nil + }) + if err != nil { + return nil, fmt.Errorf("failed to register attempt: %w", err) + } + + return mpPayment, nil +} + +// SettleAttempt marks the specified HTLC attempt as successfully settled, +// recording the payment preimage and settlement time. The preimage serves as +// cryptographic proof of payment and is atomically saved to the database. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface. It represents +// step 3a in the payment lifecycle control flow (step 3b is FailAttempt), +// called after RegisterAttempt when an HTLC successfully completes. +func (s *SQLStore) SettleAttempt(ctx context.Context, paymentHash lntypes.Hash, + attemptID uint64, settleInfo *HTLCSettleInfo) (*MPPayment, error) { + + var mpPayment *MPPayment + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + dbPayment, err := fetchPaymentByHash(ctx, db, paymentHash) + if err != nil { + return err + } + + paymentStatus, err := computePaymentStatusFromDB( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + if err := paymentStatus.updatable(); err != nil { + return fmt.Errorf("payment is not updatable: %w", err) + } + + err = db.SettleAttempt(ctx, sqlc.SettleAttemptParams{ + AttemptIndex: int64(attemptID), + ResolutionTime: time.Now(), + ResolutionType: int32(HTLCAttemptResolutionSettled), + SettlePreimage: settleInfo.Preimage[:], + }) + if err != nil { + return fmt.Errorf("failed to settle attempt: %w", err) + } + + // Fetch the complete payment after we settled the attempt. + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + return nil + }, func() { + mpPayment = nil + }) + if err != nil { + return nil, fmt.Errorf("failed to settle attempt: %w", err) + } + + return mpPayment, nil +} + +// FailAttempt marks the specified HTLC attempt as failed, recording the +// failure reason, failure time, optional failure message, and the index of the +// node in the route that generated the failure. This information is atomically +// saved to the database for debugging and route optimization purposes. +// +// For single-path payments, failing the only attempt may lead to the payment +// being retried or ultimately failed via the Fail method. For multi-shard +// (MPP/AMP) payments, individual shard failures don't necessarily fail the +// entire payment; additional attempts can be registered until sufficient shards +// succeed or the payment is permanently failed. +// +// Returns the updated MPPayment with the attempt marked as failed and the +// payment state recalculated. The payment status remains StatusInFlight if +// other attempts are still in flight, or may transition based on the overall +// payment state. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface. It represents +// step 3b in the payment lifecycle control flow (step 3a is SettleAttempt), +// called after RegisterAttempt when an HTLC fails. +func (s *SQLStore) FailAttempt(ctx context.Context, paymentHash lntypes.Hash, + attemptID uint64, failInfo *HTLCFailInfo) (*MPPayment, error) { + + var mpPayment *MPPayment + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + // Make sure the payment exists. + dbPayment, err := fetchPaymentByHash(ctx, db, paymentHash) + if err != nil { + return err + } + + paymentStatus, err := computePaymentStatusFromDB( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + // We check if the payment is updatable before failing the + // attempt. + if err := paymentStatus.updatable(); err != nil { + return fmt.Errorf("payment is not updatable: %w", err) + } + + var failureMsg bytes.Buffer + if failInfo.Message != nil { + err := lnwire.EncodeFailureMessage( + &failureMsg, failInfo.Message, 0, + ) + if err != nil { + return fmt.Errorf("failed to encode "+ + "failure message: %w", err) + } + } + + err = db.FailAttempt(ctx, sqlc.FailAttemptParams{ + AttemptIndex: int64(attemptID), + ResolutionTime: time.Now(), + ResolutionType: int32(HTLCAttemptResolutionFailed), + FailureSourceIndex: sqldb.SQLInt32( + failInfo.FailureSourceIndex, + ), + HtlcFailReason: sqldb.SQLInt32(failInfo.Reason), + FailureMsg: failureMsg.Bytes(), + }) + if err != nil { + return fmt.Errorf("failed to fail attempt: %w", err) + } + + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, dbPayment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + return nil + }, func() { + mpPayment = nil + }) + if err != nil { + return nil, fmt.Errorf("failed to fail attempt: %w", err) + } + + return mpPayment, nil +} + +// Fail records the ultimate reason why a payment failed. This method stores +// the failure reason for record keeping but does not enforce that all HTLC +// attempts are resolved - HTLCs may still be in flight when this is called. +// +// The payment's actual status transition to StatusFailed is determined by the +// payment state calculation, which considers both the recorded failure reason +// and the current state of all HTLC attempts. The status will transition to +// StatusFailed once all HTLCs are resolved and/or a failure reason is recorded. +// +// NOTE: According to the interface contract, this should only be called when +// all active attempts are already failed. However, the implementation allows +// concurrent calls and does not validate this precondition, enabling the last +// failing attempt to record the failure reason without synchronization. +// +// This method is part of the PaymentControl interface, which is embedded in +// the PaymentWriter interface and ultimately the DB interface. It represents +// step 4 in the payment lifecycle control flow. +func (s *SQLStore) Fail(ctx context.Context, paymentHash lntypes.Hash, + reason FailureReason) (*MPPayment, error) { + + var mpPayment *MPPayment + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + result, err := db.FailPayment(ctx, sqlc.FailPaymentParams{ + PaymentIdentifier: paymentHash[:], + FailReason: sqldb.SQLInt32(reason), + }) + if err != nil { + return err + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return err + } + if rowsAffected == 0 { + return ErrPaymentNotInitiated + } + + payment, err := db.FetchPayment(ctx, paymentHash[:]) + if err != nil { + return fmt.Errorf("failed to fetch payment: %w", err) + } + mpPayment, err = fetchPaymentWithCompleteData( + ctx, s.cfg.QueryCfg, db, payment, + ) + if err != nil { + return fmt.Errorf("failed to fetch payment with "+ + "complete data: %w", err) + } + + return nil + }, func() { + mpPayment = nil + }) + if err != nil { + return nil, fmt.Errorf("failed to fail payment: %w", err) + } + + return mpPayment, nil +} + +// DeletePayments performs a batch deletion of payments or their failed HTLC +// attempts from the database based on the specified flags. This is a bulk +// operation that iterates through all payments and selectively deletes based +// on the criteria. +// The behavior is controlled by two flags: +// +// If failedAttemptsOnly is true, only failed HTLC attempts are deleted while +// preserving the payment records and any successful or in-flight attempts. +// The return value is always 0 when deleting attempts only. +// +// If failedAttemptsOnly is false, entire payment records are deleted including +// all associated data (HTLCs, metadata, intents). The return value is the +// number of payments deleted. +// +// The failedOnly flag further filters which payments are processed: +// - failedOnly=true, failedAttemptsOnly=true: Delete failed attempts for +// StatusFailed payments only +// - failedOnly=false, failedAttemptsOnly=true: Delete failed attempts for +// all removable payments +// - failedOnly=true, failedAttemptsOnly=false: Delete entire payment records +// for StatusFailed payments only +// - failedOnly=false, failedAttemptsOnly=false: Delete all removable payment +// records (StatusInitiated, StatusSucceeded, StatusFailed) +// +// Safety checks applied to all operations: +// - Payments with StatusInFlight are always skipped (cannot be safely deleted +// while HTLCs are on the network) +// - The payment status must pass the removable() check +// +// Returns the number of complete payments deleted (0 if only deleting failed +// attempts). This is useful for cleanup operations, administrative maintenance, +// or freeing up database storage. +// +// This method is part of the PaymentWriter interface, which is embedded in +// the DB interface. +// +// TODO(ziggie): batch and use iterator instead, moreover we dont need to fetch +// the complete payment data for each payment, we can just fetch the payment ID +// and the resolution types to decide if the payment is removable. +func (s *SQLStore) DeletePayments(ctx context.Context, failedOnly, + failedHtlcsOnly bool) (int, error) { + + var numPayments int + + extractCursor := func(row sqlc.FilterPaymentsRow) int64 { + return row.Payment.ID + } + + err := s.db.ExecTx(ctx, sqldb.WriteTxOpt(), func(db SQLQueries) error { + // collectFunc extracts the payment ID from each payment row. + collectFunc := func(row sqlc.FilterPaymentsRow) (int64, error) { + return row.Payment.ID, nil + } + + // batchDataFunc loads only HTLC resolution types for a batch + // of payments, which is sufficient to determine payment status. + batchDataFunc := func(ctx context.Context, paymentIDs []int64) ( + *paymentStatusData, error) { + + return batchLoadPaymentResolutions( + ctx, s.cfg.QueryCfg, db, paymentIDs, + ) + } + + // processPayment processes each payment with the lightweight + // batch-loaded resolution data. + processPayment := func(ctx context.Context, + dbPayment sqlc.FilterPaymentsRow, + batchData *paymentStatusData) error { + + payment := dbPayment.Payment + + // Compute the payment status from resolution types and + // failure reason without building the complete payment. + resolutionTypes := batchData.resolutionTypes[payment.ID] + status, err := computePaymentStatusFromResolutions( + resolutionTypes, payment.FailReason, + ) + if err != nil { + return fmt.Errorf("failed to compute payment "+ + "status: %w", err) + } + + // Payments which are not final yet cannot be deleted. + // we skip them. + if err := status.removable(); err != nil { + return nil + } + + // If we are only deleting failed payments, we skip + // if the payment is not failed. + if failedOnly && status != StatusFailed { + return nil + } + + // If we are only deleting failed HTLCs, we delete them + // and return early. + if failedHtlcsOnly { + return db.DeleteFailedAttempts( + ctx, payment.ID, + ) + } + + // Otherwise we delete the payment. + err = db.DeletePayment(ctx, payment.ID) + if err != nil { + return fmt.Errorf("failed to delete "+ + "payment: %w", err) + } + + numPayments++ + + return nil + } + + queryFunc := func(ctx context.Context, lastID int64, + limit int32) ([]sqlc.FilterPaymentsRow, error) { + + filterParams := sqlc.FilterPaymentsParams{ + NumLimit: limit, + CreatedAfter: time.Unix(0, 0).UTC(), + CreatedBefore: time.Date( + 9999, 12, 31, 23, 59, 59, + 0, time.UTC, + ), + IndexOffsetGet: sqldb.SQLInt64( + lastID, + ), + } + + return db.FilterPayments(ctx, filterParams) + } + + return sqldb.ExecuteCollectAndBatchWithSharedDataQuery( + ctx, s.cfg.QueryCfg, int64(-1), queryFunc, + extractCursor, collectFunc, batchDataFunc, + processPayment, + ) + }, func() { + numPayments = 0 + }) + if err != nil { + return 0, fmt.Errorf("failed to delete payments "+ + "(failedOnly: %v, failedHtlcsOnly: %v): %w", + failedOnly, failedHtlcsOnly, err) + } + + return numPayments, nil +} diff --git a/payments/db/migration1/sqlc/db.go b/payments/db/migration1/sqlc/db.go new file mode 100644 index 000000000..e4d78283b --- /dev/null +++ b/payments/db/migration1/sqlc/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package sqlc + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/payments/db/migration1/sqlc/db_custom.go b/payments/db/migration1/sqlc/db_custom.go new file mode 100644 index 000000000..625e65adc --- /dev/null +++ b/payments/db/migration1/sqlc/db_custom.go @@ -0,0 +1,123 @@ +package sqlc + +import ( + "fmt" + "strings" +) + +// GetTx returns the underlying DBTX (either *sql.DB or *sql.Tx) used by the +// Queries struct. +func (q *Queries) GetTx() DBTX { + return q.db +} + +// makeQueryParams generates a string of query parameters for a SQL query. It is +// meant to replace the `?` placeholders in a SQL query with numbered parameters +// like `$1`, `$2`, etc. This is required for the sqlc /*SLICE:*/ +// workaround. See scripts/gen_sqlc_docker.sh for more details. +func makeQueryParams(numTotalArgs, numListArgs int) string { + if numListArgs == 0 { + return "" + } + + var b strings.Builder + + // Pre-allocate a rough estimation of the buffer size to avoid + // re-allocations. A parameter like $1000, takes 6 bytes. + b.Grow(numListArgs * 6) + + diff := numTotalArgs - numListArgs + for i := 0; i < numListArgs; i++ { + if i > 0 { + // We don't need to check the error here because the + // WriteString method of strings.Builder always returns + // nil. + _, _ = b.WriteString(",") + } + + // We don't need to check the error here because the + // Write method (called by fmt.Fprintf) of strings.Builder + // always returns nil. + _, _ = fmt.Fprintf(&b, "$%d", i+diff+1) + } + + return b.String() +} + +// PaymentAndIntent is an interface that provides access to a payment and its +// associated payment intent. +type PaymentAndIntent interface { + // GetPayment returns the Payment associated with this interface. + GetPayment() Payment + + // GetPaymentIntent returns the PaymentIntent associated with this + // payment. + GetPaymentIntent() PaymentIntent +} + +// GetPayment returns the Payment associated with this interface. +// +// NOTE: This method is part of the PaymentAndIntent interface. +func (r FilterPaymentsRow) GetPayment() Payment { + return r.Payment +} + +// GetPaymentIntent returns the PaymentIntent associated with this payment. +// If the payment has no intent (IntentType is NULL), this returns a zero-value +// PaymentIntent. +// +// NOTE: This method is part of the PaymentAndIntent interface. +func (r FilterPaymentsRow) GetPaymentIntent() PaymentIntent { + if !r.IntentType.Valid { + return PaymentIntent{} + } + + return PaymentIntent{ + IntentType: r.IntentType.Int16, + IntentPayload: r.IntentPayload, + } +} + +// GetPayment returns the Payment associated with this interface. +// +// NOTE: This method is part of the PaymentAndIntent interface. +func (r FetchPaymentRow) GetPayment() Payment { + return r.Payment +} + +// GetPaymentIntent returns the PaymentIntent associated with this payment. +// If the payment has no intent (IntentType is NULL), this returns a zero-value +// PaymentIntent. +// +// NOTE: This method is part of the PaymentAndIntent interface. +func (r FetchPaymentRow) GetPaymentIntent() PaymentIntent { + if !r.IntentType.Valid { + return PaymentIntent{} + } + + return PaymentIntent{ + IntentType: r.IntentType.Int16, + IntentPayload: r.IntentPayload, + } +} + +func (r FetchPaymentsByIDsRow) GetPayment() Payment { + return Payment{ + ID: r.ID, + AmountMsat: r.AmountMsat, + CreatedAt: r.CreatedAt, + PaymentIdentifier: r.PaymentIdentifier, + FailReason: r.FailReason, + } +} + +func (r FetchPaymentsByIDsRow) GetPaymentIntent() PaymentIntent { + if !r.IntentType.Valid { + return PaymentIntent{} + } + + return PaymentIntent{ + IntentType: r.IntentType.Int16, + IntentPayload: r.IntentPayload, + } +} diff --git a/payments/db/migration1/sqlc/models.go b/payments/db/migration1/sqlc/models.go new file mode 100644 index 000000000..afb448090 --- /dev/null +++ b/payments/db/migration1/sqlc/models.go @@ -0,0 +1,110 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 + +package sqlc + +import ( + "database/sql" + "time" +) + +type Payment struct { + ID int64 + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 +} + +type PaymentAttemptFirstHopCustomRecord struct { + ID int64 + HtlcAttemptIndex int64 + Key int64 + Value []byte +} + +type PaymentDuplicate struct { + ID int64 + PaymentID int64 + AmountMsat int64 + CreatedAt time.Time + FailReason sql.NullInt32 + SettlePreimage []byte + SettleTime sql.NullTime +} + +type PaymentFirstHopCustomRecord struct { + ID int64 + PaymentID int64 + Key int64 + Value []byte +} + +type PaymentHopCustomRecord struct { + ID int64 + HopID int64 + Key int64 + Value []byte +} + +type PaymentHtlcAttempt struct { + ID int64 + AttemptIndex int64 + PaymentID int64 + SessionKey []byte + AttemptTime time.Time + PaymentHash []byte + FirstHopAmountMsat int64 + RouteTotalTimeLock int32 + RouteTotalAmount int64 + RouteSourceKey []byte +} + +type PaymentHtlcAttemptResolution struct { + AttemptIndex int64 + ResolutionTime time.Time + ResolutionType int32 + SettlePreimage []byte + FailureSourceIndex sql.NullInt32 + HtlcFailReason sql.NullInt32 + FailureMsg []byte +} + +type PaymentIntent struct { + ID int64 + PaymentID int64 + IntentType int16 + IntentPayload []byte +} + +type PaymentRouteHop struct { + ID int64 + HtlcAttemptIndex int64 + HopIndex int32 + PubKey []byte + Scid string + OutgoingTimeLock int32 + AmtToForward int64 + MetaData []byte +} + +type PaymentRouteHopAmp struct { + HopID int64 + RootShare []byte + SetID []byte + ChildIndex int32 +} + +type PaymentRouteHopBlinded struct { + HopID int64 + EncryptedData []byte + BlindingPoint []byte + BlindedPathTotalAmt sql.NullInt64 +} + +type PaymentRouteHopMpp struct { + HopID int64 + PaymentAddr []byte + TotalMsat int64 +} diff --git a/payments/db/migration1/sqlc/payments.sql.go b/payments/db/migration1/sqlc/payments.sql.go new file mode 100644 index 000000000..e17bb3cfb --- /dev/null +++ b/payments/db/migration1/sqlc/payments.sql.go @@ -0,0 +1,1357 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: payments.sql + +package sqlc + +import ( + "context" + "database/sql" + "strings" + "time" +) + +const countPayments = `-- name: CountPayments :one +SELECT COUNT(*) FROM payments +` + +func (q *Queries) CountPayments(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countPayments) + var count int64 + err := row.Scan(&count) + return count, err +} + +const deleteFailedAttempts = `-- name: DeleteFailedAttempts :exec +DELETE FROM payment_htlc_attempts WHERE payment_id = $1 AND attempt_index IN ( + SELECT attempt_index FROM payment_htlc_attempt_resolutions WHERE resolution_type = 2 +) +` + +// Delete all failed HTLC attempts for the given payment. Resolution type 2 +// indicates a failed attempt. +func (q *Queries) DeleteFailedAttempts(ctx context.Context, paymentID int64) error { + _, err := q.db.ExecContext(ctx, deleteFailedAttempts, paymentID) + return err +} + +const deletePayment = `-- name: DeletePayment :exec +DELETE FROM payments WHERE id = $1 +` + +func (q *Queries) DeletePayment(ctx context.Context, id int64) error { + _, err := q.db.ExecContext(ctx, deletePayment, id) + return err +} + +const failAttempt = `-- name: FailAttempt :exec +INSERT INTO payment_htlc_attempt_resolutions ( + attempt_index, + resolution_time, + resolution_type, + failure_source_index, + htlc_fail_reason, + failure_msg +) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6 +) +` + +type FailAttemptParams struct { + AttemptIndex int64 + ResolutionTime time.Time + ResolutionType int32 + FailureSourceIndex sql.NullInt32 + HtlcFailReason sql.NullInt32 + FailureMsg []byte +} + +func (q *Queries) FailAttempt(ctx context.Context, arg FailAttemptParams) error { + _, err := q.db.ExecContext(ctx, failAttempt, + arg.AttemptIndex, + arg.ResolutionTime, + arg.ResolutionType, + arg.FailureSourceIndex, + arg.HtlcFailReason, + arg.FailureMsg, + ) + return err +} + +const failPayment = `-- name: FailPayment :execresult +UPDATE payments SET fail_reason = $1 WHERE payment_identifier = $2 +` + +type FailPaymentParams struct { + FailReason sql.NullInt32 + PaymentIdentifier []byte +} + +func (q *Queries) FailPayment(ctx context.Context, arg FailPaymentParams) (sql.Result, error) { + return q.db.ExecContext(ctx, failPayment, arg.FailReason, arg.PaymentIdentifier) +} + +const fetchAllInflightAttempts = `-- name: FetchAllInflightAttempts :many +SELECT + ha.id, + ha.attempt_index, + ha.payment_id, + ha.session_key, + ha.attempt_time, + ha.payment_hash, + ha.first_hop_amount_msat, + ha.route_total_time_lock, + ha.route_total_amount, + ha.route_source_key +FROM payment_htlc_attempts ha +WHERE NOT EXISTS ( + SELECT 1 FROM payment_htlc_attempt_resolutions hr + WHERE hr.attempt_index = ha.attempt_index +) +AND ha.attempt_index > $1 +ORDER BY ha.attempt_index ASC +LIMIT $2 +` + +type FetchAllInflightAttemptsParams struct { + AttemptIndex int64 + Limit int32 +} + +// Fetch all inflight attempts with their payment data using pagination. +// Returns attempt data joined with payment and intent data to avoid separate queries. +func (q *Queries) FetchAllInflightAttempts(ctx context.Context, arg FetchAllInflightAttemptsParams) ([]PaymentHtlcAttempt, error) { + rows, err := q.db.QueryContext(ctx, fetchAllInflightAttempts, arg.AttemptIndex, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentHtlcAttempt + for rows.Next() { + var i PaymentHtlcAttempt + if err := rows.Scan( + &i.ID, + &i.AttemptIndex, + &i.PaymentID, + &i.SessionKey, + &i.AttemptTime, + &i.PaymentHash, + &i.FirstHopAmountMsat, + &i.RouteTotalTimeLock, + &i.RouteTotalAmount, + &i.RouteSourceKey, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchHopLevelCustomRecords = `-- name: FetchHopLevelCustomRecords :many +SELECT + l.id, + l.hop_id, + l.key, + l.value +FROM payment_hop_custom_records l +WHERE l.hop_id IN (/*SLICE:hop_ids*/?) +ORDER BY l.hop_id ASC, l.key ASC +` + +func (q *Queries) FetchHopLevelCustomRecords(ctx context.Context, hopIds []int64) ([]PaymentHopCustomRecord, error) { + query := fetchHopLevelCustomRecords + var queryParams []interface{} + if len(hopIds) > 0 { + for _, v := range hopIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:hop_ids*/?", makeQueryParams(len(queryParams), len(hopIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:hop_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentHopCustomRecord + for rows.Next() { + var i PaymentHopCustomRecord + if err := rows.Scan( + &i.ID, + &i.HopID, + &i.Key, + &i.Value, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchHopsForAttempts = `-- name: FetchHopsForAttempts :many +SELECT + h.id, + h.htlc_attempt_index, + h.hop_index, + h.pub_key, + h.scid, + h.outgoing_time_lock, + h.amt_to_forward, + h.meta_data, + m.payment_addr AS mpp_payment_addr, + m.total_msat AS mpp_total_msat, + a.root_share AS amp_root_share, + a.set_id AS amp_set_id, + a.child_index AS amp_child_index, + b.encrypted_data, + b.blinding_point, + b.blinded_path_total_amt +FROM payment_route_hops h +LEFT JOIN payment_route_hop_mpp m ON m.hop_id = h.id +LEFT JOIN payment_route_hop_amp a ON a.hop_id = h.id +LEFT JOIN payment_route_hop_blinded b ON b.hop_id = h.id +WHERE h.htlc_attempt_index IN (/*SLICE:htlc_attempt_indices*/?) +ORDER BY h.htlc_attempt_index ASC, h.hop_index ASC +` + +type FetchHopsForAttemptsRow struct { + ID int64 + HtlcAttemptIndex int64 + HopIndex int32 + PubKey []byte + Scid string + OutgoingTimeLock int32 + AmtToForward int64 + MetaData []byte + MppPaymentAddr []byte + MppTotalMsat sql.NullInt64 + AmpRootShare []byte + AmpSetID []byte + AmpChildIndex sql.NullInt32 + EncryptedData []byte + BlindingPoint []byte + BlindedPathTotalAmt sql.NullInt64 +} + +func (q *Queries) FetchHopsForAttempts(ctx context.Context, htlcAttemptIndices []int64) ([]FetchHopsForAttemptsRow, error) { + query := fetchHopsForAttempts + var queryParams []interface{} + if len(htlcAttemptIndices) > 0 { + for _, v := range htlcAttemptIndices { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:htlc_attempt_indices*/?", makeQueryParams(len(queryParams), len(htlcAttemptIndices)), 1) + } else { + query = strings.Replace(query, "/*SLICE:htlc_attempt_indices*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchHopsForAttemptsRow + for rows.Next() { + var i FetchHopsForAttemptsRow + if err := rows.Scan( + &i.ID, + &i.HtlcAttemptIndex, + &i.HopIndex, + &i.PubKey, + &i.Scid, + &i.OutgoingTimeLock, + &i.AmtToForward, + &i.MetaData, + &i.MppPaymentAddr, + &i.MppTotalMsat, + &i.AmpRootShare, + &i.AmpSetID, + &i.AmpChildIndex, + &i.EncryptedData, + &i.BlindingPoint, + &i.BlindedPathTotalAmt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchHtlcAttemptResolutionsForPayments = `-- name: FetchHtlcAttemptResolutionsForPayments :many +SELECT + ha.payment_id, + hr.resolution_type +FROM payment_htlc_attempts ha +LEFT JOIN payment_htlc_attempt_resolutions hr ON hr.attempt_index = ha.attempt_index +WHERE ha.payment_id IN (/*SLICE:payment_ids*/?) +` + +type FetchHtlcAttemptResolutionsForPaymentsRow struct { + PaymentID int64 + ResolutionType sql.NullInt32 +} + +// Batch query to fetch only HTLC resolution status for multiple payments. +// We don't need to order by payment_id and attempt_time because we will +// group the resolutions by payment_id in the background. +func (q *Queries) FetchHtlcAttemptResolutionsForPayments(ctx context.Context, paymentIds []int64) ([]FetchHtlcAttemptResolutionsForPaymentsRow, error) { + query := fetchHtlcAttemptResolutionsForPayments + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchHtlcAttemptResolutionsForPaymentsRow + for rows.Next() { + var i FetchHtlcAttemptResolutionsForPaymentsRow + if err := rows.Scan(&i.PaymentID, &i.ResolutionType); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchHtlcAttemptsForPayments = `-- name: FetchHtlcAttemptsForPayments :many +SELECT + ha.id, + ha.attempt_index, + ha.payment_id, + ha.session_key, + ha.attempt_time, + ha.payment_hash, + ha.first_hop_amount_msat, + ha.route_total_time_lock, + ha.route_total_amount, + ha.route_source_key, + hr.resolution_type, + hr.resolution_time, + hr.failure_source_index, + hr.htlc_fail_reason, + hr.failure_msg, + hr.settle_preimage +FROM payment_htlc_attempts ha +LEFT JOIN payment_htlc_attempt_resolutions hr ON hr.attempt_index = ha.attempt_index +WHERE ha.payment_id IN (/*SLICE:payment_ids*/?) +ORDER BY ha.payment_id ASC, ha.attempt_time ASC +` + +type FetchHtlcAttemptsForPaymentsRow struct { + ID int64 + AttemptIndex int64 + PaymentID int64 + SessionKey []byte + AttemptTime time.Time + PaymentHash []byte + FirstHopAmountMsat int64 + RouteTotalTimeLock int32 + RouteTotalAmount int64 + RouteSourceKey []byte + ResolutionType sql.NullInt32 + ResolutionTime sql.NullTime + FailureSourceIndex sql.NullInt32 + HtlcFailReason sql.NullInt32 + FailureMsg []byte + SettlePreimage []byte +} + +func (q *Queries) FetchHtlcAttemptsForPayments(ctx context.Context, paymentIds []int64) ([]FetchHtlcAttemptsForPaymentsRow, error) { + query := fetchHtlcAttemptsForPayments + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchHtlcAttemptsForPaymentsRow + for rows.Next() { + var i FetchHtlcAttemptsForPaymentsRow + if err := rows.Scan( + &i.ID, + &i.AttemptIndex, + &i.PaymentID, + &i.SessionKey, + &i.AttemptTime, + &i.PaymentHash, + &i.FirstHopAmountMsat, + &i.RouteTotalTimeLock, + &i.RouteTotalAmount, + &i.RouteSourceKey, + &i.ResolutionType, + &i.ResolutionTime, + &i.FailureSourceIndex, + &i.HtlcFailReason, + &i.FailureMsg, + &i.SettlePreimage, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchPayment = `-- name: FetchPayment :one +SELECT + p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason, + i.intent_type AS "intent_type", + i.intent_payload AS "intent_payload" +FROM payments p +LEFT JOIN payment_intents i ON i.payment_id = p.id +WHERE p.payment_identifier = $1 +` + +type FetchPaymentRow struct { + Payment Payment + IntentType sql.NullInt16 + IntentPayload []byte +} + +func (q *Queries) FetchPayment(ctx context.Context, paymentIdentifier []byte) (FetchPaymentRow, error) { + row := q.db.QueryRowContext(ctx, fetchPayment, paymentIdentifier) + var i FetchPaymentRow + err := row.Scan( + &i.Payment.ID, + &i.Payment.AmountMsat, + &i.Payment.CreatedAt, + &i.Payment.PaymentIdentifier, + &i.Payment.FailReason, + &i.IntentType, + &i.IntentPayload, + ) + return i, err +} + +const fetchPaymentDuplicates = `-- name: FetchPaymentDuplicates :many +SELECT + id, + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +FROM payment_duplicates +WHERE payment_id = $1 +ORDER BY id ASC +` + +// Fetch all duplicate payment records from the payment_duplicates table for +// a given payment ID. +func (q *Queries) FetchPaymentDuplicates(ctx context.Context, paymentID int64) ([]PaymentDuplicate, error) { + rows, err := q.db.QueryContext(ctx, fetchPaymentDuplicates, paymentID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentDuplicate + for rows.Next() { + var i PaymentDuplicate + if err := rows.Scan( + &i.ID, + &i.PaymentID, + &i.AmountMsat, + &i.CreatedAt, + &i.FailReason, + &i.SettlePreimage, + &i.SettleTime, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchPaymentLevelFirstHopCustomRecords = `-- name: FetchPaymentLevelFirstHopCustomRecords :many +SELECT + l.id, + l.payment_id, + l.key, + l.value +FROM payment_first_hop_custom_records l +WHERE l.payment_id IN (/*SLICE:payment_ids*/?) +ORDER BY l.payment_id ASC, l.key ASC +` + +func (q *Queries) FetchPaymentLevelFirstHopCustomRecords(ctx context.Context, paymentIds []int64) ([]PaymentFirstHopCustomRecord, error) { + query := fetchPaymentLevelFirstHopCustomRecords + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentFirstHopCustomRecord + for rows.Next() { + var i PaymentFirstHopCustomRecord + if err := rows.Scan( + &i.ID, + &i.PaymentID, + &i.Key, + &i.Value, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchPaymentsByIDs = `-- name: FetchPaymentsByIDs :many +SELECT + p.id, + p.amount_msat, + p.created_at, + p.payment_identifier, + p.fail_reason, + pi.intent_type, + pi.intent_payload +FROM payments p +LEFT JOIN payment_intents pi ON pi.payment_id = p.id +WHERE p.id IN (/*SLICE:payment_ids*/?) +ORDER BY p.id ASC +` + +type FetchPaymentsByIDsRow struct { + ID int64 + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 + IntentType sql.NullInt16 + IntentPayload []byte +} + +// Batch fetch payment and intent data for a set of payment IDs. +// Used to avoid fetching redundant payment data when processing multiple +// attempts for the same payment. +func (q *Queries) FetchPaymentsByIDs(ctx context.Context, paymentIds []int64) ([]FetchPaymentsByIDsRow, error) { + query := fetchPaymentsByIDs + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchPaymentsByIDsRow + for rows.Next() { + var i FetchPaymentsByIDsRow + if err := rows.Scan( + &i.ID, + &i.AmountMsat, + &i.CreatedAt, + &i.PaymentIdentifier, + &i.FailReason, + &i.IntentType, + &i.IntentPayload, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchPaymentsByIDsMig = `-- name: FetchPaymentsByIDsMig :many +SELECT + p.id, + p.amount_msat, + p.created_at, + p.payment_identifier, + p.fail_reason, + COUNT(ha.id) AS htlc_attempt_count +FROM payments p +LEFT JOIN payment_htlc_attempts ha ON ha.payment_id = p.id +WHERE p.id IN (/*SLICE:payment_ids*/?) +GROUP BY p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason +ORDER BY p.id ASC +` + +type FetchPaymentsByIDsMigRow struct { + ID int64 + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 + HtlcAttemptCount int64 +} + +// Migration-specific batch fetch that returns payment data along with HTLC +// attempt counts for structural validation during KV to SQL migration. +func (q *Queries) FetchPaymentsByIDsMig(ctx context.Context, paymentIds []int64) ([]FetchPaymentsByIDsMigRow, error) { + query := fetchPaymentsByIDsMig + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchPaymentsByIDsMigRow + for rows.Next() { + var i FetchPaymentsByIDsMigRow + if err := rows.Scan( + &i.ID, + &i.AmountMsat, + &i.CreatedAt, + &i.PaymentIdentifier, + &i.FailReason, + &i.HtlcAttemptCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const fetchRouteLevelFirstHopCustomRecords = `-- name: FetchRouteLevelFirstHopCustomRecords :many +SELECT + l.id, + l.htlc_attempt_index, + l.key, + l.value +FROM payment_attempt_first_hop_custom_records l +WHERE l.htlc_attempt_index IN (/*SLICE:htlc_attempt_indices*/?) +ORDER BY l.htlc_attempt_index ASC, l.key ASC +` + +func (q *Queries) FetchRouteLevelFirstHopCustomRecords(ctx context.Context, htlcAttemptIndices []int64) ([]PaymentAttemptFirstHopCustomRecord, error) { + query := fetchRouteLevelFirstHopCustomRecords + var queryParams []interface{} + if len(htlcAttemptIndices) > 0 { + for _, v := range htlcAttemptIndices { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:htlc_attempt_indices*/?", makeQueryParams(len(queryParams), len(htlcAttemptIndices)), 1) + } else { + query = strings.Replace(query, "/*SLICE:htlc_attempt_indices*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentAttemptFirstHopCustomRecord + for rows.Next() { + var i PaymentAttemptFirstHopCustomRecord + if err := rows.Scan( + &i.ID, + &i.HtlcAttemptIndex, + &i.Key, + &i.Value, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const filterPayments = `-- name: FilterPayments :many +/* ───────────────────────────────────────────── + fetch queries + ───────────────────────────────────────────── +*/ + +SELECT + p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason, + i.intent_type AS "intent_type", + i.intent_payload AS "intent_payload" +FROM payments p +LEFT JOIN payment_intents i ON i.payment_id = p.id +WHERE p.id > COALESCE($1, -1) + AND p.id < COALESCE($2, 9223372036854775807) + -- NOTE: We use non-nullable time params with Go-side defaults instead of + -- COALESCE, because COALESCE with text fallback causes type mismatch on + -- Postgres (timestamp vs text), and OR-based optional filters can prevent + -- the planner from using the created_at index. + AND p.created_at >= $3 + AND p.created_at <= $4 + AND ( + i.intent_type = $5 OR + $5 IS NULL OR i.intent_type IS NULL + ) +ORDER BY p.id ASC +LIMIT $6 +` + +type FilterPaymentsParams struct { + IndexOffsetGet sql.NullInt64 + IndexOffsetLet sql.NullInt64 + CreatedAfter time.Time + CreatedBefore time.Time + IntentType sql.NullInt16 + NumLimit int32 +} + +type FilterPaymentsRow struct { + Payment Payment + IntentType sql.NullInt16 + IntentPayload []byte +} + +func (q *Queries) FilterPayments(ctx context.Context, arg FilterPaymentsParams) ([]FilterPaymentsRow, error) { + rows, err := q.db.QueryContext(ctx, filterPayments, + arg.IndexOffsetGet, + arg.IndexOffsetLet, + arg.CreatedAfter, + arg.CreatedBefore, + arg.IntentType, + arg.NumLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FilterPaymentsRow + for rows.Next() { + var i FilterPaymentsRow + if err := rows.Scan( + &i.Payment.ID, + &i.Payment.AmountMsat, + &i.Payment.CreatedAt, + &i.Payment.PaymentIdentifier, + &i.Payment.FailReason, + &i.IntentType, + &i.IntentPayload, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const filterPaymentsDesc = `-- name: FilterPaymentsDesc :many +SELECT + p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason, + i.intent_type AS "intent_type", + i.intent_payload AS "intent_payload" +FROM payments p +LEFT JOIN payment_intents i ON i.payment_id = p.id +WHERE p.id > COALESCE($1, -1) + AND p.id < COALESCE($2, 9223372036854775807) + -- NOTE: We use non-nullable time params with Go-side defaults instead of + -- COALESCE, because COALESCE with text fallback causes type mismatch on + -- Postgres (timestamp vs text), and OR-based optional filters can prevent + -- the planner from using the created_at index. + AND p.created_at >= $3 + AND p.created_at <= $4 + AND ( + i.intent_type = $5 OR + $5 IS NULL OR i.intent_type IS NULL + ) +ORDER BY p.id DESC +LIMIT $6 +` + +type FilterPaymentsDescParams struct { + IndexOffsetGet sql.NullInt64 + IndexOffsetLet sql.NullInt64 + CreatedAfter time.Time + CreatedBefore time.Time + IntentType sql.NullInt16 + NumLimit int32 +} + +type FilterPaymentsDescRow struct { + Payment Payment + IntentType sql.NullInt16 + IntentPayload []byte +} + +func (q *Queries) FilterPaymentsDesc(ctx context.Context, arg FilterPaymentsDescParams) ([]FilterPaymentsDescRow, error) { + rows, err := q.db.QueryContext(ctx, filterPaymentsDesc, + arg.IndexOffsetGet, + arg.IndexOffsetLet, + arg.CreatedAfter, + arg.CreatedBefore, + arg.IntentType, + arg.NumLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FilterPaymentsDescRow + for rows.Next() { + var i FilterPaymentsDescRow + if err := rows.Scan( + &i.Payment.ID, + &i.Payment.AmountMsat, + &i.Payment.CreatedAt, + &i.Payment.PaymentIdentifier, + &i.Payment.FailReason, + &i.IntentType, + &i.IntentPayload, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const insertHtlcAttempt = `-- name: InsertHtlcAttempt :one +INSERT INTO payment_htlc_attempts ( + payment_id, + attempt_index, + session_key, + attempt_time, + payment_hash, + first_hop_amount_msat, + route_total_time_lock, + route_total_amount, + route_source_key) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9) +RETURNING id +` + +type InsertHtlcAttemptParams struct { + PaymentID int64 + AttemptIndex int64 + SessionKey []byte + AttemptTime time.Time + PaymentHash []byte + FirstHopAmountMsat int64 + RouteTotalTimeLock int32 + RouteTotalAmount int64 + RouteSourceKey []byte +} + +func (q *Queries) InsertHtlcAttempt(ctx context.Context, arg InsertHtlcAttemptParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertHtlcAttempt, + arg.PaymentID, + arg.AttemptIndex, + arg.SessionKey, + arg.AttemptTime, + arg.PaymentHash, + arg.FirstHopAmountMsat, + arg.RouteTotalTimeLock, + arg.RouteTotalAmount, + arg.RouteSourceKey, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertPayment = `-- name: InsertPayment :one +INSERT INTO payments ( + amount_msat, + created_at, + payment_identifier, + fail_reason) +VALUES ( + $1, + $2, + $3, + NULL +) +RETURNING id +` + +type InsertPaymentParams struct { + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte +} + +// Insert a new payment and return its ID. +// When creating a payment we don't have a fail reason because we start the +// payment process. +func (q *Queries) InsertPayment(ctx context.Context, arg InsertPaymentParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPayment, arg.AmountMsat, arg.CreatedAt, arg.PaymentIdentifier) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertPaymentAttemptFirstHopCustomRecord = `-- name: InsertPaymentAttemptFirstHopCustomRecord :exec +INSERT INTO payment_attempt_first_hop_custom_records ( + htlc_attempt_index, + key, + value +) +VALUES ( + $1, + $2, + $3 +) +` + +type InsertPaymentAttemptFirstHopCustomRecordParams struct { + HtlcAttemptIndex int64 + Key int64 + Value []byte +} + +func (q *Queries) InsertPaymentAttemptFirstHopCustomRecord(ctx context.Context, arg InsertPaymentAttemptFirstHopCustomRecordParams) error { + _, err := q.db.ExecContext(ctx, insertPaymentAttemptFirstHopCustomRecord, arg.HtlcAttemptIndex, arg.Key, arg.Value) + return err +} + +const insertPaymentDuplicateMig = `-- name: InsertPaymentDuplicateMig :one +INSERT INTO payment_duplicates ( + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6 +) +RETURNING id +` + +type InsertPaymentDuplicateMigParams struct { + PaymentID int64 + AmountMsat int64 + CreatedAt time.Time + FailReason sql.NullInt32 + SettlePreimage []byte + SettleTime sql.NullTime +} + +// Insert a duplicate payment record into the payment_duplicates table and +// return its ID. +func (q *Queries) InsertPaymentDuplicateMig(ctx context.Context, arg InsertPaymentDuplicateMigParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPaymentDuplicateMig, + arg.PaymentID, + arg.AmountMsat, + arg.CreatedAt, + arg.FailReason, + arg.SettlePreimage, + arg.SettleTime, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertPaymentFirstHopCustomRecord = `-- name: InsertPaymentFirstHopCustomRecord :exec +INSERT INTO payment_first_hop_custom_records ( + payment_id, + key, + value +) +VALUES ( + $1, + $2, + $3 +) +` + +type InsertPaymentFirstHopCustomRecordParams struct { + PaymentID int64 + Key int64 + Value []byte +} + +func (q *Queries) InsertPaymentFirstHopCustomRecord(ctx context.Context, arg InsertPaymentFirstHopCustomRecordParams) error { + _, err := q.db.ExecContext(ctx, insertPaymentFirstHopCustomRecord, arg.PaymentID, arg.Key, arg.Value) + return err +} + +const insertPaymentHopCustomRecord = `-- name: InsertPaymentHopCustomRecord :exec +INSERT INTO payment_hop_custom_records ( + hop_id, + key, + value +) +VALUES ( + $1, + $2, + $3 +) +` + +type InsertPaymentHopCustomRecordParams struct { + HopID int64 + Key int64 + Value []byte +} + +func (q *Queries) InsertPaymentHopCustomRecord(ctx context.Context, arg InsertPaymentHopCustomRecordParams) error { + _, err := q.db.ExecContext(ctx, insertPaymentHopCustomRecord, arg.HopID, arg.Key, arg.Value) + return err +} + +const insertPaymentIntent = `-- name: InsertPaymentIntent :one +INSERT INTO payment_intents ( + payment_id, + intent_type, + intent_payload) +VALUES ( + $1, + $2, + $3 +) +RETURNING id +` + +type InsertPaymentIntentParams struct { + PaymentID int64 + IntentType int16 + IntentPayload []byte +} + +// Insert a payment intent for a given payment and return its ID. +func (q *Queries) InsertPaymentIntent(ctx context.Context, arg InsertPaymentIntentParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPaymentIntent, arg.PaymentID, arg.IntentType, arg.IntentPayload) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertPaymentMig = `-- name: InsertPaymentMig :one +/* ───────────────────────────────────────────── + Migration-specific queries + + These queries are used ONLY for the one-time migration from KV to SQL. + ───────────────────────────────────────────── +*/ + +INSERT INTO payments ( + amount_msat, + created_at, + payment_identifier, + fail_reason) +VALUES ( + $1, + $2, + $3, + $4 +) +RETURNING id +` + +type InsertPaymentMigParams struct { + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 +} + +// Migration-specific payment insert that allows setting fail_reason. +// Normal InsertPayment forces fail_reason to NULL since new payments +// aren't failed yet. During migration, we're inserting historical data +// that may already be failed. +func (q *Queries) InsertPaymentMig(ctx context.Context, arg InsertPaymentMigParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPaymentMig, + arg.AmountMsat, + arg.CreatedAt, + arg.PaymentIdentifier, + arg.FailReason, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertRouteHop = `-- name: InsertRouteHop :one +INSERT INTO payment_route_hops ( + htlc_attempt_index, + hop_index, + pub_key, + scid, + outgoing_time_lock, + amt_to_forward, + meta_data +) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7 +) +RETURNING id +` + +type InsertRouteHopParams struct { + HtlcAttemptIndex int64 + HopIndex int32 + PubKey []byte + Scid string + OutgoingTimeLock int32 + AmtToForward int64 + MetaData []byte +} + +func (q *Queries) InsertRouteHop(ctx context.Context, arg InsertRouteHopParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertRouteHop, + arg.HtlcAttemptIndex, + arg.HopIndex, + arg.PubKey, + arg.Scid, + arg.OutgoingTimeLock, + arg.AmtToForward, + arg.MetaData, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertRouteHopAmp = `-- name: InsertRouteHopAmp :exec +INSERT INTO payment_route_hop_amp ( + hop_id, + root_share, + set_id, + child_index +) +VALUES ( + $1, + $2, + $3, + $4 +) +` + +type InsertRouteHopAmpParams struct { + HopID int64 + RootShare []byte + SetID []byte + ChildIndex int32 +} + +func (q *Queries) InsertRouteHopAmp(ctx context.Context, arg InsertRouteHopAmpParams) error { + _, err := q.db.ExecContext(ctx, insertRouteHopAmp, + arg.HopID, + arg.RootShare, + arg.SetID, + arg.ChildIndex, + ) + return err +} + +const insertRouteHopBlinded = `-- name: InsertRouteHopBlinded :exec +INSERT INTO payment_route_hop_blinded ( + hop_id, + encrypted_data, + blinding_point, + blinded_path_total_amt +) +VALUES ( + $1, + $2, + $3, + $4 +) +` + +type InsertRouteHopBlindedParams struct { + HopID int64 + EncryptedData []byte + BlindingPoint []byte + BlindedPathTotalAmt sql.NullInt64 +} + +func (q *Queries) InsertRouteHopBlinded(ctx context.Context, arg InsertRouteHopBlindedParams) error { + _, err := q.db.ExecContext(ctx, insertRouteHopBlinded, + arg.HopID, + arg.EncryptedData, + arg.BlindingPoint, + arg.BlindedPathTotalAmt, + ) + return err +} + +const insertRouteHopMpp = `-- name: InsertRouteHopMpp :exec +INSERT INTO payment_route_hop_mpp ( + hop_id, + payment_addr, + total_msat +) +VALUES ( + $1, + $2, + $3 +) +` + +type InsertRouteHopMppParams struct { + HopID int64 + PaymentAddr []byte + TotalMsat int64 +} + +func (q *Queries) InsertRouteHopMpp(ctx context.Context, arg InsertRouteHopMppParams) error { + _, err := q.db.ExecContext(ctx, insertRouteHopMpp, arg.HopID, arg.PaymentAddr, arg.TotalMsat) + return err +} + +const settleAttempt = `-- name: SettleAttempt :exec +INSERT INTO payment_htlc_attempt_resolutions ( + attempt_index, + resolution_time, + resolution_type, + settle_preimage +) +VALUES ( + $1, + $2, + $3, + $4 +) +` + +type SettleAttemptParams struct { + AttemptIndex int64 + ResolutionTime time.Time + ResolutionType int32 + SettlePreimage []byte +} + +func (q *Queries) SettleAttempt(ctx context.Context, arg SettleAttemptParams) error { + _, err := q.db.ExecContext(ctx, settleAttempt, + arg.AttemptIndex, + arg.ResolutionTime, + arg.ResolutionType, + arg.SettlePreimage, + ) + return err +} diff --git a/payments/db/migration1/test_harness.go b/payments/db/migration1/test_harness.go new file mode 100644 index 000000000..b25867a4f --- /dev/null +++ b/payments/db/migration1/test_harness.go @@ -0,0 +1,26 @@ +package migration1 + +import ( + "testing" + + "github.com/lightningnetwork/lnd/lntypes" +) + +// TestHarness provides implementation-specific test utilities for the payments +// database. Different database backends (KV, SQL) have different internal +// structures and indexing mechanisms, so this interface allows tests to verify +// implementation-specific behavior without coupling the test logic to a +// particular backend. +type TestHarness interface { + // AssertPaymentIndex checks that a payment is correctly indexed. + // For KV: verifies the payment index bucket entry exists and points + // to the correct payment hash. + // For SQL: no-op (SQL doesn't use a separate index bucket). + AssertPaymentIndex(t *testing.T, expectedHash lntypes.Hash) + + // AssertNoIndex checks that an index for a sequence number doesn't + // exist. + // For KV: verifies the index bucket entry is deleted. + // For SQL: no-op. + AssertNoIndex(t *testing.T, seqNr uint64) +} diff --git a/payments/db/migration1/test_postgres.go b/payments/db/migration1/test_postgres.go new file mode 100644 index 000000000..7055fb885 --- /dev/null +++ b/payments/db/migration1/test_postgres.go @@ -0,0 +1,94 @@ +//go:build test_db_postgres && !test_db_sqlite + +package migration1 + +import ( + "testing" + + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// NewTestDB is a helper function that creates a SQLStore backed by a SQL +// database for testing. +func NewTestDB(t testing.TB, opts ...OptionModifier) (DB, TestHarness) { + db := NewTestDBWithFixture(t, nil, opts...) + return db, &noopTestHarness{} +} + +// NewTestDBFixture creates a new sqldb.TestPgFixture for testing purposes. +func NewTestDBFixture(t *testing.T) *sqldb.TestPgFixture { + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + return pgFixture +} + +// NewTestDBWithFixture is a helper function that creates a SQLStore backed by a +// SQL database for testing. +func NewTestDBWithFixture(t testing.TB, + pgFixture *sqldb.TestPgFixture, opts ...OptionModifier) DB { + + var querier BatchedSQLQueries + if pgFixture == nil { + querier = newBatchQuerier(t) + } else { + querier = newBatchQuerierWithFixture(t, pgFixture) + } + + store, err := NewSQLStore( + &SQLStoreConfig{ + QueryCfg: sqldb.DefaultPostgresConfig(), + }, querier, opts..., + ) + require.NoError(t, err) + + return store +} + +// newBatchQuerier creates a new BatchedSQLQueries instance for testing +// using a PostgreSQL database fixture. +func newBatchQuerier(t testing.TB) BatchedSQLQueries { + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + + return newBatchQuerierWithFixture(t, pgFixture) +} + +// newBatchQuerierWithFixture creates a new BatchedSQLQueries instance for +// testing using a PostgreSQL database fixture. +func newBatchQuerierWithFixture(t testing.TB, + pgFixture *sqldb.TestPgFixture) BatchedSQLQueries { + + rawDB := sqldb.NewTestPostgresDB(t, pgFixture).BaseDB.DB + + return &testBatchedSQLQueries{ + db: rawDB, + Queries: sqlc.New(rawDB), + } +} + +// noopTestHarness is the SQL test harness implementation. Since SQL doesn't +// use a separate payment index bucket like KV, these assertions are no-ops. +type noopTestHarness struct{} + +// AssertPaymentIndex is a no-op for SQL implementations. +func (h *noopTestHarness) AssertPaymentIndex(t *testing.T, + expectedHash lntypes.Hash) { + + // No-op: SQL doesn't use a separate index bucket. +} + +// AssertNoIndex is a no-op for SQL implementations. +func (h *noopTestHarness) AssertNoIndex(t *testing.T, seqNr uint64) { + // No-op: SQL doesn't use a separate index bucket. +} diff --git a/payments/db/migration1/test_sql.go b/payments/db/migration1/test_sql.go new file mode 100644 index 000000000..f2d4d078b --- /dev/null +++ b/payments/db/migration1/test_sql.go @@ -0,0 +1,58 @@ +//go:build test_db_postgres || test_db_sqlite + +package migration1 + +import ( + "context" + "database/sql" + "testing" + + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// setupTestSQLDB creates a SQLStore-backed test database. +func setupTestSQLDB(t testing.TB, opts ...OptionModifier) *SQLStore { + t.Helper() + + db, _ := NewTestDB(t, opts...) + sqlStore, ok := db.(*SQLStore) + require.True(t, ok) + + return sqlStore +} + +// testBatchedSQLQueries is a simple implementation of BatchedSQLQueries for +// testing. +type testBatchedSQLQueries struct { + db *sql.DB + *sqlc.Queries +} + +// ExecTx implements the transaction execution logic. +func (t *testBatchedSQLQueries) ExecTx(ctx context.Context, + txOpts sqldb.TxOptions, txBody func(SQLQueries) error, + reset func()) error { + + sqlOptions := sql.TxOptions{ + Isolation: sql.LevelSerializable, + ReadOnly: txOpts.ReadOnly(), + } + + tx, err := t.db.BeginTx(ctx, &sqlOptions) + if err != nil { + return err + } + + reset() + queries := sqlc.New(tx) + + if err := txBody(queries); err != nil { + _ = tx.Rollback() + + return err + } + + return tx.Commit() +} diff --git a/payments/db/migration1/test_sqlite.go b/payments/db/migration1/test_sqlite.go new file mode 100644 index 000000000..b84c9c2d9 --- /dev/null +++ b/payments/db/migration1/test_sqlite.go @@ -0,0 +1,73 @@ +//go:build !test_db_postgres && test_db_sqlite + +package migration1 + +import ( + "testing" + + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +// NewTestDB is a helper function that creates a SQLStore backed by a SQL +// database for testing. +func NewTestDB(t testing.TB, opts ...OptionModifier) (DB, TestHarness) { + db := NewTestDBWithFixture(t, nil, opts...) + return db, &noopTestHarness{} +} + +// NewTestDBFixture is a no-op for the sqlite build. +func NewTestDBFixture(_ *testing.T) *sqldb.TestPgFixture { + return nil +} + +// NewTestDBWithFixture is a helper function that creates a SQLStore backed by a +// SQL database for testing. +func NewTestDBWithFixture(t testing.TB, _ *sqldb.TestPgFixture, + opts ...OptionModifier) DB { + + store, err := NewSQLStore( + &SQLStoreConfig{ + QueryCfg: sqldb.DefaultSQLiteConfig(), + }, newBatchQuerier(t), opts..., + ) + require.NoError(t, err) + return store +} + +// newBatchQuerier creates a new BatchedSQLQueries instance for testing +// using a SQLite database. +func newBatchQuerier(t testing.TB) BatchedSQLQueries { + return newBatchQuerierWithFixture(t, nil) +} + +// newBatchQuerierWithFixture creates a new BatchedSQLQueries instance for +// testing using a SQLite database. +func newBatchQuerierWithFixture(t testing.TB, + _ *sqldb.TestPgFixture) BatchedSQLQueries { + + rawDB := sqldb.NewTestSqliteDB(t).BaseDB.DB + + return &testBatchedSQLQueries{ + db: rawDB, + Queries: sqlc.New(rawDB), + } +} + +// noopTestHarness is the SQL test harness implementation. Since SQL doesn't +// use a separate payment index bucket like KV, these assertions are no-ops. +type noopTestHarness struct{} + +// AssertPaymentIndex is a no-op for SQL implementations. +func (h *noopTestHarness) AssertPaymentIndex(t *testing.T, + expectedHash lntypes.Hash) { + + // No-op: SQL doesn't use a separate index bucket. +} + +// AssertNoIndex is a no-op for SQL implementations. +func (h *noopTestHarness) AssertNoIndex(t *testing.T, seqNr uint64) { + // No-op: SQL doesn't use a separate index bucket. +} diff --git a/payments/db/migration1/testdata/README.md b/payments/db/migration1/testdata/README.md new file mode 100644 index 000000000..08c5fb06b --- /dev/null +++ b/payments/db/migration1/testdata/README.md @@ -0,0 +1,52 @@ +# Payment Migration External Testdata + +This directory holds a real `channel.db` (bbolt) or `channel.sqlite` file for +testing the payments KV to SQL migration locally. You can also point the test +at an existing Postgres-backed kvdb instance. + +## How to use + +1. Copy your `channel.db` or `channel.sqlite` file into this folder. +2. Edit `migration_external_test.go`: + + ```go + // Comment out this line to enable the test + t.Skipf("skipping test meant for local debugging only") + + // Set to your database filename + const fileName = "channel.db" // or "channel.sqlite" + ``` + +3. Run the test: + + ```bash + # For Postgres backend + go test -v -tags="test_db_postgres" -run TestMigrationWithExternalDB + ``` + +## SQLite kvdb source + +To migrate from a `channel.sqlite` file, run with the `kvdb_sqlite` build +tag: + +```bash +go test -v -tags="test_db_sqlite kvdb_sqlite" \ + -run TestMigrationWithExternalDB +``` + +## Postgres kvdb source + +To migrate from an existing Postgres-backed kvdb instance, edit +`postgresKVDSN` in `migration_external_test.go` (set it non-empty), then +run with the `kvdb_postgres` build tag: + +```bash +go test -v -tags="kvdb_postgres test_db_postgres" \ + -run TestMigrationWithExternalDB +``` + +## Notes + +- The external database is opened read-only. +- The test creates a fresh SQL database for each run. +- Do not commit production data; keep the file local. diff --git a/payments/db/sql_store.go b/payments/db/sql_store.go index 3637a981b..1c8ffb581 100644 --- a/payments/db/sql_store.go +++ b/payments/db/sql_store.go @@ -57,6 +57,8 @@ type SQLQueries interface { FetchAllInflightAttempts(ctx context.Context, arg sqlc.FetchAllInflightAttemptsParams) ([]sqlc.PaymentHtlcAttempt, error) FetchHopsForAttempts(ctx context.Context, htlcAttemptIndices []int64) ([]sqlc.FetchHopsForAttemptsRow, error) + FetchPaymentDuplicates(ctx context.Context, paymentID int64) ([]sqlc.PaymentDuplicate, error) + FetchPaymentLevelFirstHopCustomRecords(ctx context.Context, paymentIDs []int64) ([]sqlc.PaymentFirstHopCustomRecord, error) FetchRouteLevelFirstHopCustomRecords(ctx context.Context, htlcAttemptIndices []int64) ([]sqlc.PaymentAttemptFirstHopCustomRecord, error) FetchHopLevelCustomRecords(ctx context.Context, hopIDs []int64) ([]sqlc.PaymentHopCustomRecord, error) @@ -89,6 +91,31 @@ type SQLQueries interface { DeleteFailedAttempts(ctx context.Context, paymentID int64) error } +// SQLMigrationQueries extends SQLQueries with the additional queries needed +// for the one-time migration from KV to SQL. Keeping them in a separate +// interface makes it clear which code paths are migration-only and prevents +// the regular store from accidentally depending on them. +// +//nolint:ll +type SQLMigrationQueries interface { + SQLQueries + + // FetchPaymentsByIDsMig is a migration-only batch fetch that returns + // payment data along with HTLC attempt counts for structural + // validation. + FetchPaymentsByIDsMig(ctx context.Context, paymentIDs []int64) ([]sqlc.FetchPaymentsByIDsMigRow, error) + + // InsertPaymentMig is a migration-only variant of InsertPayment that + // allows setting fail_reason when inserting historical payments, since + // for real payments they have not failed at creation time and so no + // failure reason would exist yet. + InsertPaymentMig(ctx context.Context, arg sqlc.InsertPaymentMigParams) (int64, error) + + // InsertPaymentDuplicateMig inserts a duplicate payment record during + // migration. + InsertPaymentDuplicateMig(ctx context.Context, arg sqlc.InsertPaymentDuplicateMigParams) (int64, error) +} + // BatchedSQLQueries is a version of the SQLQueries that's capable // of batched database operations. type BatchedSQLQueries interface { diff --git a/sqldb/migrations_dev.go b/sqldb/migrations_dev.go index ca57cf0f1..7b02b151f 100644 --- a/sqldb/migrations_dev.go +++ b/sqldb/migrations_dev.go @@ -8,4 +8,18 @@ var migrationAdditions = []MigrationConfig{ Version: 12, SchemaVersion: 10, }, + { + Name: "000011_payment_duplicates", + Version: 13, + SchemaVersion: 11, + }, + { + Name: "kv_payments_migration", + Version: 14, + SchemaVersion: 11, + // A migration function may be attached to this + // migration to migrate KV payments to the native SQL + // schema. This is optional and can be disabled by the + // user if necessary. + }, } diff --git a/sqldb/migrations_test.go b/sqldb/migrations_test.go index 68f111420..399afb1ce 100644 --- a/sqldb/migrations_test.go +++ b/sqldb/migrations_test.go @@ -616,3 +616,100 @@ func TestMigrationSucceedsAfterDirtyStateMigrationFailure19RC1(t *testing.T) { require.False(t, dirty) }) } + +// TestMigrationConfigConsistency verifies that the migration configuration in +// migrationConfig is consistent with the actual SQL schema files embedded in +// the binary. This catches version collisions (e.g. two migrations claiming +// the same schema version) and missing schema files. +func TestMigrationConfigConsistency(t *testing.T) { + t.Parallel() + + migrations := GetMigrations() + require.NotEmpty(t, migrations) + + // Build a set of schema versions that have actual .up.sql files in + // the embedded filesystem. + embeddedFiles, err := sqlSchemas.ReadDir("sqlc/migrations") + require.NoError(t, err) + + fileSchemaVersions := make(map[int]string) + for _, f := range embeddedFiles { + if f.IsDir() { + continue + } + + var version int + _, err := fmt.Sscanf(f.Name(), "%06d_", &version) + require.NoError(t, err, "schema migration file %q is "+ + "missing a valid numeric prefix (expected "+ + "format: 000XXX_name.up.sql)", f.Name()) + + // Enforce the 6-digit zero-padded naming convention + // for consistent directory listing order. + expectedPrefix := fmt.Sprintf("%06d_", version) + require.True(t, + len(f.Name()) > len(expectedPrefix) && + f.Name()[:len(expectedPrefix)] == expectedPrefix, + "schema migration file %q should use 6-digit "+ + "zero-padded prefix %q", f.Name(), + expectedPrefix) + + // Verify no two files share the same numeric prefix. + if existing, ok := fileSchemaVersions[version]; ok { + t.Fatalf("duplicate schema file version %06d: "+ + "%q and %q", version, existing, f.Name()) + } + + fileSchemaVersions[version] = f.Name() + } + + // Track seen versions to detect duplicates. + seenVersions := make(map[int]string) + seenSchemaVersions := make(map[int]string) + + for _, m := range migrations { + // 1. Verify no duplicate global versions. + if existing, ok := seenVersions[m.Version]; ok { + t.Fatalf("duplicate global version %d: %q and %q", + m.Version, existing, m.Name) + } + seenVersions[m.Version] = m.Name + + // 2. For schema migrations (those that advance the schema + // version), verify a corresponding .up.sql file exists + // and no two config entries claim the same schema version + // with different file prefixes. + prevSchema := 0 + if m.Version > 1 { + prevSchema = migrations[m.Version-2].SchemaVersion + } + + // A migration advances the schema if its SchemaVersion is + // higher than the previous migration's SchemaVersion. + if m.SchemaVersion > prevSchema { + _, hasFile := fileSchemaVersions[m.SchemaVersion] + require.True(t, hasFile, + "migration %q (version %d) declares "+ + "SchemaVersion=%d but no %06d_*.up.sql"+ + " file exists in the embedded FS", + m.Name, m.Version, m.SchemaVersion, + m.SchemaVersion) + + if existing, ok := seenSchemaVersions[m.SchemaVersion]; ok { + t.Fatalf("duplicate schema version %d: "+ + "%q and %q", m.SchemaVersion, + existing, m.Name) + } + seenSchemaVersions[m.SchemaVersion] = m.Name + } + } + + // 3. Verify versions are sequential starting from 1. + for i, m := range migrations { + require.Equal(t, i+1, m.Version, + "migration %q has version %d but expected %d "+ + "(migrations must be sequential)", + m.Name, m.Version, i+1) + } + +} diff --git a/sqldb/sqlc/migrations/000011_payment_duplicates.down.sql b/sqldb/sqlc/migrations/000011_payment_duplicates.down.sql new file mode 100644 index 000000000..39c4a313a --- /dev/null +++ b/sqldb/sqlc/migrations/000011_payment_duplicates.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_payment_duplicates_payment_id; +DROP TABLE IF EXISTS payment_duplicates; diff --git a/sqldb/sqlc/migrations/000011_payment_duplicates.up.sql b/sqldb/sqlc/migrations/000011_payment_duplicates.up.sql new file mode 100644 index 000000000..8abaf3e09 --- /dev/null +++ b/sqldb/sqlc/migrations/000011_payment_duplicates.up.sql @@ -0,0 +1,43 @@ +-- ───────────────────────────────────────────── +-- Payment Duplicate Records Table +-- ───────────────────────────────────────────── +-- Stores duplicate payment records that were created in older versions +-- of lnd. This table is intentionally minimal and is expected to be dropped +-- in the future especially if no duplicates were migrated. +-- ───────────────────────────────────────────── + +CREATE TABLE IF NOT EXISTS payment_duplicates ( + -- Primary key for the duplicate record. + id INTEGER PRIMARY KEY, + + -- Reference to the primary payment this duplicate belongs to. + payment_id BIGINT NOT NULL REFERENCES payments (id) ON DELETE CASCADE, + + -- Amount of the duplicate payment in millisatoshis. + amount_msat BIGINT NOT NULL, + + -- Timestamp when the duplicate payment was created. + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + + -- Failure reason for failed payments (if known). + fail_reason INTEGER, + + -- Settlement payload for succeeded payments (if known). + settle_preimage BLOB, + + -- Settlement time for succeeded payments (if known). + settle_time TIMESTAMP, + + -- Ensure we record either a failure reason or settlement data. + -- During the migration if we encounter a duplicate payment that has no + -- failure reason or settlement data, we will mark it as failed. Duplicate + -- payments were a bug in older versions of LND, so we can be sure if a + -- duplicate payment has no failure reason or settlement data, the + -- corresponding HTLC has been failed. + CONSTRAINT chk_payment_duplicates_outcome + CHECK (fail_reason IS NOT NULL OR settle_preimage IS NOT NULL) +); + +-- Index for efficient lookup by primary payment. +CREATE INDEX IF NOT EXISTS idx_payment_duplicates_payment_id +ON payment_duplicates(payment_id); diff --git a/sqldb/sqlc/models.go b/sqldb/sqlc/models.go index 0e04cd467..513f045ba 100644 --- a/sqldb/sqlc/models.go +++ b/sqldb/sqlc/models.go @@ -224,6 +224,16 @@ type PaymentAttemptFirstHopCustomRecord struct { Value []byte } +type PaymentDuplicate struct { + ID int64 + PaymentID int64 + AmountMsat int64 + CreatedAt time.Time + FailReason sql.NullInt32 + SettlePreimage []byte + SettleTime sql.NullTime +} + type PaymentFirstHopCustomRecord struct { ID int64 PaymentID int64 diff --git a/sqldb/sqlc/payments.sql.go b/sqldb/sqlc/payments.sql.go index b9ec31499..5ac3d826a 100644 --- a/sqldb/sqlc/payments.sql.go +++ b/sqldb/sqlc/payments.sql.go @@ -479,6 +479,53 @@ func (q *Queries) FetchPayment(ctx context.Context, paymentIdentifier []byte) (F return i, err } +const fetchPaymentDuplicates = `-- name: FetchPaymentDuplicates :many +SELECT + id, + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +FROM payment_duplicates +WHERE payment_id = $1 +ORDER BY id ASC +` + +// Fetch all duplicate payment records from the payment_duplicates table for +// a given payment ID. +func (q *Queries) FetchPaymentDuplicates(ctx context.Context, paymentID int64) ([]PaymentDuplicate, error) { + rows, err := q.db.QueryContext(ctx, fetchPaymentDuplicates, paymentID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []PaymentDuplicate + for rows.Next() { + var i PaymentDuplicate + if err := rows.Scan( + &i.ID, + &i.PaymentID, + &i.AmountMsat, + &i.CreatedAt, + &i.FailReason, + &i.SettlePreimage, + &i.SettleTime, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const fetchPaymentLevelFirstHopCustomRecords = `-- name: FetchPaymentLevelFirstHopCustomRecords :many SELECT l.id, @@ -597,6 +644,72 @@ func (q *Queries) FetchPaymentsByIDs(ctx context.Context, paymentIds []int64) ([ return items, nil } +const fetchPaymentsByIDsMig = `-- name: FetchPaymentsByIDsMig :many +SELECT + p.id, + p.amount_msat, + p.created_at, + p.payment_identifier, + p.fail_reason, + COUNT(ha.id) AS htlc_attempt_count +FROM payments p +LEFT JOIN payment_htlc_attempts ha ON ha.payment_id = p.id +WHERE p.id IN (/*SLICE:payment_ids*/?) +GROUP BY p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason +ORDER BY p.id ASC +` + +type FetchPaymentsByIDsMigRow struct { + ID int64 + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 + HtlcAttemptCount int64 +} + +// Migration-specific batch fetch that returns payment data along with HTLC +// attempt counts for structural validation during KV to SQL migration. +func (q *Queries) FetchPaymentsByIDsMig(ctx context.Context, paymentIds []int64) ([]FetchPaymentsByIDsMigRow, error) { + query := fetchPaymentsByIDsMig + var queryParams []interface{} + if len(paymentIds) > 0 { + for _, v := range paymentIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:payment_ids*/?", makeQueryParams(len(queryParams), len(paymentIds)), 1) + } else { + query = strings.Replace(query, "/*SLICE:payment_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []FetchPaymentsByIDsMigRow + for rows.Next() { + var i FetchPaymentsByIDsMigRow + if err := rows.Scan( + &i.ID, + &i.AmountMsat, + &i.CreatedAt, + &i.PaymentIdentifier, + &i.FailReason, + &i.HtlcAttemptCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const fetchRouteLevelFirstHopCustomRecords = `-- name: FetchRouteLevelFirstHopCustomRecords :many SELECT l.id, @@ -843,6 +956,51 @@ func (q *Queries) InsertPaymentAttemptFirstHopCustomRecord(ctx context.Context, return err } +const insertPaymentDuplicateMig = `-- name: InsertPaymentDuplicateMig :one +INSERT INTO payment_duplicates ( + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6 +) +RETURNING id +` + +type InsertPaymentDuplicateMigParams struct { + PaymentID int64 + AmountMsat int64 + CreatedAt time.Time + FailReason sql.NullInt32 + SettlePreimage []byte + SettleTime sql.NullTime +} + +// Insert a duplicate payment record into the payment_duplicates table and +// return its ID. +func (q *Queries) InsertPaymentDuplicateMig(ctx context.Context, arg InsertPaymentDuplicateMigParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPaymentDuplicateMig, + arg.PaymentID, + arg.AmountMsat, + arg.CreatedAt, + arg.FailReason, + arg.SettlePreimage, + arg.SettleTime, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + const insertPaymentFirstHopCustomRecord = `-- name: InsertPaymentFirstHopCustomRecord :exec INSERT INTO payment_first_hop_custom_records ( payment_id, @@ -918,6 +1076,51 @@ func (q *Queries) InsertPaymentIntent(ctx context.Context, arg InsertPaymentInte return id, err } +const insertPaymentMig = `-- name: InsertPaymentMig :one +/* ───────────────────────────────────────────── + Migration-specific queries + + These queries are used ONLY for the one-time migration from KV to SQL. + ───────────────────────────────────────────── +*/ + +INSERT INTO payments ( + amount_msat, + created_at, + payment_identifier, + fail_reason) +VALUES ( + $1, + $2, + $3, + $4 +) +RETURNING id +` + +type InsertPaymentMigParams struct { + AmountMsat int64 + CreatedAt time.Time + PaymentIdentifier []byte + FailReason sql.NullInt32 +} + +// Migration-specific payment insert that allows setting fail_reason. +// Normal InsertPayment forces fail_reason to NULL since new payments +// aren't failed yet. During migration, we're inserting historical data +// that may already be failed. +func (q *Queries) InsertPaymentMig(ctx context.Context, arg InsertPaymentMigParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertPaymentMig, + arg.AmountMsat, + arg.CreatedAt, + arg.PaymentIdentifier, + arg.FailReason, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + const insertRouteHop = `-- name: InsertRouteHop :one INSERT INTO payment_route_hops ( htlc_attempt_index, diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 3be738ba9..947237300 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -49,11 +49,17 @@ type Querier interface { FetchHtlcAttemptResolutionsForPayments(ctx context.Context, paymentIds []int64) ([]FetchHtlcAttemptResolutionsForPaymentsRow, error) FetchHtlcAttemptsForPayments(ctx context.Context, paymentIds []int64) ([]FetchHtlcAttemptsForPaymentsRow, error) FetchPayment(ctx context.Context, paymentIdentifier []byte) (FetchPaymentRow, error) + // Fetch all duplicate payment records from the payment_duplicates table for + // a given payment ID. + FetchPaymentDuplicates(ctx context.Context, paymentID int64) ([]PaymentDuplicate, error) FetchPaymentLevelFirstHopCustomRecords(ctx context.Context, paymentIds []int64) ([]PaymentFirstHopCustomRecord, error) // Batch fetch payment and intent data for a set of payment IDs. // Used to avoid fetching redundant payment data when processing multiple // attempts for the same payment. FetchPaymentsByIDs(ctx context.Context, paymentIds []int64) ([]FetchPaymentsByIDsRow, error) + // Migration-specific batch fetch that returns payment data along with HTLC + // attempt counts for structural validation during KV to SQL migration. + FetchPaymentsByIDsMig(ctx context.Context, paymentIds []int64) ([]FetchPaymentsByIDsMigRow, error) // FetchPendingInvoices returns all invoices in a pending state (open or // accepted). The invoices_state_idx index on the state column makes this a // fast index scan rather than a full table scan. @@ -174,10 +180,18 @@ type Querier interface { // payment process. InsertPayment(ctx context.Context, arg InsertPaymentParams) (int64, error) InsertPaymentAttemptFirstHopCustomRecord(ctx context.Context, arg InsertPaymentAttemptFirstHopCustomRecordParams) error + // Insert a duplicate payment record into the payment_duplicates table and + // return its ID. + InsertPaymentDuplicateMig(ctx context.Context, arg InsertPaymentDuplicateMigParams) (int64, error) InsertPaymentFirstHopCustomRecord(ctx context.Context, arg InsertPaymentFirstHopCustomRecordParams) error InsertPaymentHopCustomRecord(ctx context.Context, arg InsertPaymentHopCustomRecordParams) error // Insert a payment intent for a given payment and return its ID. InsertPaymentIntent(ctx context.Context, arg InsertPaymentIntentParams) (int64, error) + // Migration-specific payment insert that allows setting fail_reason. + // Normal InsertPayment forces fail_reason to NULL since new payments + // aren't failed yet. During migration, we're inserting historical data + // that may already be failed. + InsertPaymentMig(ctx context.Context, arg InsertPaymentMigParams) (int64, error) InsertRouteHop(ctx context.Context, arg InsertRouteHopParams) (int64, error) InsertRouteHopAmp(ctx context.Context, arg InsertRouteHopAmpParams) error InsertRouteHopBlinded(ctx context.Context, arg InsertRouteHopBlindedParams) error diff --git a/sqldb/sqlc/queries/payments.sql b/sqldb/sqlc/queries/payments.sql index 419f7bf1a..1bb2c50ff 100644 --- a/sqldb/sqlc/queries/payments.sql +++ b/sqldb/sqlc/queries/payments.sql @@ -40,6 +40,21 @@ FROM payments p LEFT JOIN payment_intents i ON i.payment_id = p.id WHERE p.payment_identifier = $1; +-- name: FetchPaymentDuplicates :many +-- Fetch all duplicate payment records from the payment_duplicates table for +-- a given payment ID. +SELECT + id, + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +FROM payment_duplicates +WHERE payment_id = $1 +ORDER BY id ASC; + -- name: CountPayments :one SELECT COUNT(*) FROM payments; @@ -368,3 +383,65 @@ VALUES ( -- name: FailPayment :execresult UPDATE payments SET fail_reason = $1 WHERE payment_identifier = $2; + +/* ───────────────────────────────────────────── + Migration-specific queries + + These queries are used ONLY for the one-time migration from KV to SQL. + ───────────────────────────────────────────── +*/ + +-- name: InsertPaymentMig :one +-- Migration-specific payment insert that allows setting fail_reason. +-- Normal InsertPayment forces fail_reason to NULL since new payments +-- aren't failed yet. During migration, we're inserting historical data +-- that may already be failed. +INSERT INTO payments ( + amount_msat, + created_at, + payment_identifier, + fail_reason) +VALUES ( + @amount_msat, + @created_at, + @payment_identifier, + @fail_reason +) +RETURNING id; + +-- name: FetchPaymentsByIDsMig :many +-- Migration-specific batch fetch that returns payment data along with HTLC +-- attempt counts for structural validation during KV to SQL migration. +SELECT + p.id, + p.amount_msat, + p.created_at, + p.payment_identifier, + p.fail_reason, + COUNT(ha.id) AS htlc_attempt_count +FROM payments p +LEFT JOIN payment_htlc_attempts ha ON ha.payment_id = p.id +WHERE p.id IN (sqlc.slice('payment_ids')/*SLICE:payment_ids*/) +GROUP BY p.id, p.amount_msat, p.created_at, p.payment_identifier, p.fail_reason +ORDER BY p.id ASC; + +-- name: InsertPaymentDuplicateMig :one +-- Insert a duplicate payment record into the payment_duplicates table and +-- return its ID. +INSERT INTO payment_duplicates ( + payment_id, + amount_msat, + created_at, + fail_reason, + settle_preimage, + settle_time +) +VALUES ( + @payment_id, + @amount_msat, + @created_at, + @fail_reason, + @settle_preimage, + @settle_time +) +RETURNING id;