diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d51ca9c32..deceba636 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -252,6 +252,10 @@ jobs: args: backend=bitcoind dbbackend=postgres - name: bitcoind-sqlite args: backend=bitcoind dbbackend=sqlite + - name: bitcoind-postgres-nativesql + args: backend=bitcoind dbbackend=postgres nativesql=true + - name: bitcoind-sqlite-nativesql + args: backend=bitcoind dbbackend=sqlite nativesql=true - name: neutrino args: backend=neutrino steps: diff --git a/config_builder.go b/config_builder.go index 27e9f47f6..5195777e1 100644 --- a/config_builder.go +++ b/config_builder.go @@ -3,6 +3,7 @@ package lnd import ( "bytes" "context" + "database/sql" "errors" "fmt" "io/ioutil" @@ -31,6 +32,7 @@ import ( "github.com/lightningnetwork/lnd/chainntnfs" "github.com/lightningnetwork/lnd/chainreg" "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/clock" "github.com/lightningnetwork/lnd/invoices" "github.com/lightningnetwork/lnd/keychain" "github.com/lightningnetwork/lnd/kvdb" @@ -42,6 +44,7 @@ import ( "github.com/lightningnetwork/lnd/macaroons" "github.com/lightningnetwork/lnd/rpcperms" "github.com/lightningnetwork/lnd/signal" + "github.com/lightningnetwork/lnd/sqldb" "github.com/lightningnetwork/lnd/walletunlocker" "github.com/lightningnetwork/lnd/watchtower" "github.com/lightningnetwork/lnd/watchtower/wtclient" @@ -869,6 +872,11 @@ type DatabaseInstances struct { // WalletDB is the configuration for loading the wallet database using // the btcwallet's loader. WalletDB btcwallet.LoaderOption + + // NativeSQLStore is a pointer to a native SQL store that can be used + // for native SQL queries for tables that already support it. This may + // be nil if the use-native-sql flag was not set. + NativeSQLStore *sqldb.BaseDB } // DefaultDatabaseBuilder is a type that builds the default database backends @@ -923,10 +931,11 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( // state DB point to the same local or remote DB and the same namespace // within that DB. dbs := &DatabaseInstances{ - HeightHintDB: databaseBackends.HeightHintDB, - MacaroonDB: databaseBackends.MacaroonDB, - DecayedLogDB: databaseBackends.DecayedLogDB, - WalletDB: databaseBackends.WalletDB, + HeightHintDB: databaseBackends.HeightHintDB, + MacaroonDB: databaseBackends.MacaroonDB, + DecayedLogDB: databaseBackends.DecayedLogDB, + WalletDB: databaseBackends.WalletDB, + NativeSQLStore: databaseBackends.NativeSQLStore, } cleanUp := func() { // We can just close the returned close functions directly. Even @@ -1011,11 +1020,21 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( // using the same struct (and DB backend) instance. dbs.ChanStateDB = dbs.GraphDB - // For now the only InvoiceDB implementation is the *channeldb.DB. - // - // TODO(positiveblue): use a sql first implementation for this - // interface. - dbs.InvoiceDB = dbs.GraphDB + // Instantiate a native SQL invoice store if the flag is set. + if d.cfg.DB.UseNativeSQL { + executor := sqldb.NewTransactionExecutor( + dbs.NativeSQLStore, + func(tx *sql.Tx) sqldb.InvoiceQueries { + return dbs.NativeSQLStore.WithTx(tx) + }, + ) + + dbs.InvoiceDB = sqldb.NewInvoiceStore( + executor, clock.NewDefaultClock(), + ) + } else { + dbs.InvoiceDB = dbs.GraphDB + } // Wrap the watchtower client DB and make sure we clean up. if cfg.WtClient.Active { @@ -1298,8 +1317,9 @@ func initNeutrinoBackend(ctx context.Context, cfg *Config, chainDir string, ) switch { case cfg.DB.Backend == kvdb.SqliteBackendName: + sqliteConfig := lncfg.GetSqliteConfigKVDB(cfg.DB.Sqlite) db, err = kvdb.Open( - kvdb.SqliteBackendName, ctx, cfg.DB.Sqlite, dbPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, dbPath, lncfg.SqliteNeutrinoDBName, lncfg.NSNeutrinoDB, ) diff --git a/docs/release-notes/release-notes-0.18.0.md b/docs/release-notes/release-notes-0.18.0.md index b12f35c88..6ea9bcd03 100644 --- a/docs/release-notes/release-notes-0.18.0.md +++ b/docs/release-notes/release-notes-0.18.0.md @@ -339,6 +339,10 @@ * [Refactor UpdateInvoice](https://github.com/lightningnetwork/lnd/pull/8100) to make it simpler to adjust code to also support SQL InvoiceDB implementation. +* [InvoiceDB implementation](https://github.com/lightningnetwork/lnd/pull/8052) + for SQL backends enabling new users to optionally use an experimental native + SQL invoices database. + ## Code Health * [Remove database pointers](https://github.com/lightningnetwork/lnd/pull/8117) diff --git a/invoices/invoiceregistry_test.go b/invoices/invoiceregistry_test.go index 04fe310b5..99ad6cddd 100644 --- a/invoices/invoiceregistry_test.go +++ b/invoices/invoiceregistry_test.go @@ -3,6 +3,7 @@ package invoices_test import ( "context" "crypto/rand" + "database/sql" "fmt" "math" "testing" @@ -18,6 +19,7 @@ import ( "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/sqldb" "github.com/stretchr/testify/require" ) @@ -114,12 +116,60 @@ func TestInvoiceRegistry(t *testing.T) { return db, testClock } + // First create a shared Postgres instance so we don't spawn a new + // docker container for each test. + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + + makeSQLDB := func(t *testing.T, sqlite bool) (invpkg.InvoiceDB, + *clock.TestClock) { + + var db *sqldb.BaseDB + if sqlite { + db = sqldb.NewTestSqliteDB(t).BaseDB + } else { + db = sqldb.NewTestPostgresDB(t, pgFixture).BaseDB + } + + executor := sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) sqldb.InvoiceQueries { + return db.WithTx(tx) + }, + ) + + testClock := clock.NewTestClock(testNow) + + return sqldb.NewInvoiceStore(executor, testClock), testClock + } + for _, test := range testList { test := test - t.Run(test.name, func(t *testing.T) { + t.Run(test.name+"_KV", func(t *testing.T) { test.test(t, makeKeyValueDB) }) + + t.Run(test.name+"_SQLite", func(t *testing.T) { + test.test(t, + func(t *testing.T) ( + invpkg.InvoiceDB, *clock.TestClock) { + + return makeSQLDB(t, true) + }) + }) + + t.Run(test.name+"_Postgres", func(t *testing.T) { + test.test(t, + func(t *testing.T) ( + invpkg.InvoiceDB, *clock.TestClock) { + + return makeSQLDB(t, false) + }) + }) } } diff --git a/invoices/invoices_test.go b/invoices/invoices_test.go index caf58f13a..7e1baadd3 100644 --- a/invoices/invoices_test.go +++ b/invoices/invoices_test.go @@ -3,8 +3,10 @@ package invoices_test import ( "context" "crypto/rand" + "database/sql" "fmt" "math" + "strings" "testing" "time" @@ -16,6 +18,7 @@ import ( "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/sqldb" "github.com/stretchr/testify/require" ) @@ -34,6 +37,23 @@ var ( testNow = time.Unix(1, 0) ) +// randBytesToString will return a "safe" string from a byte slice. This is +// used to generate random strings for the invoice payment request. +func randBytesToString(buf []byte) (string, error) { + const charset = "abcdefghijklmnopqrstuvwxyz0123456789" + var stringBuilder strings.Builder + + stringBuilder.Grow(len(buf)) + for i := 0; i < len(buf); i++ { + ch := charset[int(buf[i])%len(charset)] + if err := stringBuilder.WriteByte(ch); err != nil { + return "", err + } + } + + return stringBuilder.String(), nil +} + func randInvoice(value lnwire.MilliSatoshi) (*invpkg.Invoice, error) { var ( pre lntypes.Preimage @@ -68,7 +88,11 @@ func randInvoice(value lnwire.MilliSatoshi) (*invpkg.Invoice, error) { return nil, err } if r[0]&1 == 0 { - i.PaymentRequest = r[:] + paymentReq, err := randBytesToString(r[:]) + if err != nil { + return nil, err + } + i.PaymentRequest = []byte(paymentReq) } else { i.PaymentRequest = []byte("") } @@ -77,11 +101,13 @@ func randInvoice(value lnwire.MilliSatoshi) (*invpkg.Invoice, error) { } // settleTestInvoice settles a test invoice. -func settleTestInvoice(invoice *invpkg.Invoice, settleIndex uint64) { +func settleTestInvoice(invoice *invpkg.Invoice, htlcID uint64, + settleIndex uint64) { + invoice.SettleDate = testNow invoice.AmtPaid = invoice.Terms.Value invoice.State = invpkg.ContractSettled - invoice.Htlcs[models.CircuitKey{}] = &invpkg.InvoiceHTLC{ + invoice.Htlcs[models.CircuitKey{HtlcID: htlcID}] = &invpkg.InvoiceHTLC{ Amt: invoice.Terms.Value, AcceptTime: testNow, ResolveTime: testNow, @@ -196,12 +222,53 @@ func TestInvoices(t *testing.T) { return db } + // First create a shared Postgres instance so we don't spawn a new + // docker container for each test. + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + + makeSQLDB := func(t *testing.T, sqlite bool) invpkg.InvoiceDB { + var db *sqldb.BaseDB + if sqlite { + db = sqldb.NewTestSqliteDB(t).BaseDB + } else { + db = sqldb.NewTestPostgresDB(t, pgFixture).BaseDB + } + + executor := sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) sqldb.InvoiceQueries { + return db.WithTx(tx) + }, + ) + + testClock := clock.NewTestClock(testNow) + + return sqldb.NewInvoiceStore(executor, testClock) + } + for _, test := range testList { test := test - - t.Run(test.name, func(t *testing.T) { + t.Run(test.name+"_KV", func(t *testing.T) { test.test(t, makeKeyValueDB) }) + + t.Run(test.name+"_SQLite", func(t *testing.T) { + test.test(t, + func(t *testing.T) invpkg.InvoiceDB { + return makeSQLDB(t, true) + }) + }) + + t.Run(test.name+"_Postgres", func(t *testing.T) { + test.test(t, + func(t *testing.T) invpkg.InvoiceDB { + return makeSQLDB(t, false) + }) + }) } } @@ -333,7 +400,7 @@ func testInvoiceWorkflowImpl(t *testing.T, test invWorkflowTest, // now have the settled bit toggle to true and a non-default // SettledDate payAmt := fakeInvoice.Terms.Value * 2 - _, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(payAmt)) + _, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(0, payAmt)) require.NoError(t, err, "unable to settle invoice") dbInvoice2, err := db.LookupInvoice(ctxb, ref) require.NoError(t, err, "unable to fetch invoice") @@ -512,7 +579,7 @@ func testFailInvoiceLookupMPPPayAddrOnly(t *testing.T, // for the given HTLC. ref := invpkg.InvoiceRefByHashAndAddr(payHash, payAddr) _, err = db.LookupInvoice(ctxb, ref) - require.Equal(t, invpkg.ErrInvoiceNotFound, err) + require.ErrorIs(t, err, invpkg.ErrInvoiceNotFound) } // testInvRefEquivocation asserts that retrieving or updating an invoice using @@ -690,7 +757,7 @@ func testInvoiceCancelSingleHtlcAMP(t *testing.T, // At this point, we should detect that 3k satoshis total has been // paid. - require.Equal(t, dbInvoice.AmtPaid, amt*3) + require.EqualValues(t, amt*3, dbInvoice.AmtPaid) callback := func( invoice *invpkg.Invoice) (*invpkg.InvoiceUpdateDesc, error) { @@ -902,8 +969,11 @@ func testInvoiceAddTimeSeries(t *testing.T, _, err = db.InvoicesSettledSince(ctxb, 0) require.NoError(t, err) - var settledInvoices []invpkg.Invoice - var settleIndex uint64 = 1 + var ( + settledInvoices []invpkg.Invoice + settleIndex uint64 = 1 + htlcID uint64 = 0 + ) // We'll now only settle the latter half of each of those invoices. for i := 10; i < len(invoices); i++ { invoice := &invoices[i] @@ -912,15 +982,18 @@ func testInvoiceAddTimeSeries(t *testing.T, ref := invpkg.InvoiceRefByHash(paymentHash) _, err := db.UpdateInvoice( - ctxb, ref, nil, getUpdateInvoice(invoice.Terms.Value), + ctxb, ref, nil, getUpdateInvoice( + htlcID, invoice.Terms.Value, + ), ) if err != nil { t.Fatalf("unable to settle invoice: %v", err) } // Create the settled invoice for the expectation set. - settleTestInvoice(invoice, settleIndex) + settleTestInvoice(invoice, htlcID, settleIndex) settleIndex++ + htlcID++ settledInvoices = append(settledInvoices, *invoice) } @@ -960,7 +1033,10 @@ func testInvoiceAddTimeSeries(t *testing.T, t.Fatalf("unable to query: %v", err) } - require.Equal(t, len(query.resp), len(resp)) + require.Equal( + t, len(query.resp), len(resp), + fmt.Sprintf("test: #%v", i), + ) for j := 0; j < len(query.resp); j++ { require.Equal(t, @@ -1138,7 +1214,11 @@ func testSettleIndexAmpPayments(t *testing.T, AddIndex: testInvoice.AddIndex, }, }) - require.Nil(t, err) + require.NoError(t, err) + + settledInvoices, err = db.InvoicesSettledSince(ctxb, 0) + require.NoError(t, err) + require.Len(t, settledInvoices, 0) } // testFetchPendingInvoices tests that we can fetch all pending invoices from @@ -1158,7 +1238,11 @@ func testFetchPendingInvoices(t *testing.T, require.Empty(t, pending) const numInvoices = 20 - var settleIndex uint64 = 1 + var ( + settleIndex uint64 = 1 + htlcID uint64 = 0 + ) + pendingInvoices := make(map[lntypes.Hash]invpkg.Invoice) for i := 1; i <= numInvoices; i++ { @@ -1182,11 +1266,14 @@ func testFetchPendingInvoices(t *testing.T, } ref := invpkg.InvoiceRefByHash(paymentHash) - _, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(amt)) + _, err = db.UpdateInvoice( + ctxb, ref, nil, getUpdateInvoice(htlcID, amt), + ) require.NoError(t, err) - settleTestInvoice(invoice, settleIndex) + settleTestInvoice(invoice, htlcID, settleIndex) settleIndex++ + htlcID++ } // Fetch all pending invoices. @@ -1219,7 +1306,7 @@ func testDuplicateSettleInvoice(t *testing.T, // With the invoice in the DB, we'll now attempt to settle the invoice. ref := invpkg.InvoiceRefByHash(payHash) dbInvoice, err := db.UpdateInvoice( - ctxb, ref, nil, getUpdateInvoice(amt), + ctxb, ref, nil, getUpdateInvoice(0, amt), ) require.NoError(t, err, "unable to settle invoice") @@ -1244,7 +1331,9 @@ func testDuplicateSettleInvoice(t *testing.T, // If we try to settle the invoice again, then we should get the very // same invoice back, but with an error this time. - dbInvoice, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(amt)) + dbInvoice, err = db.UpdateInvoice( + ctxb, ref, nil, getUpdateInvoice(1, amt), + ) require.ErrorIs(t, err, invpkg.ErrInvoiceAlreadySettled) if dbInvoice == nil { @@ -1269,9 +1358,12 @@ func testQueryInvoices(t *testing.T, // assume that the index of the invoice within the database is the same // as the amount of the invoice itself. const numInvoices = 50 - var settleIndex uint64 = 1 - var invoices []invpkg.Invoice - var pendingInvoices []invpkg.Invoice + var ( + settleIndex uint64 = 1 + htlcID uint64 = 0 + invoices []invpkg.Invoice + pendingInvoices []invpkg.Invoice + ) ctxb := context.Background() for i := 1; i <= numInvoices; i++ { @@ -1296,15 +1388,16 @@ func testQueryInvoices(t *testing.T, if i%2 == 0 { ref := invpkg.InvoiceRefByHash(paymentHash) _, err := db.UpdateInvoice( - ctxb, ref, nil, getUpdateInvoice(amt), + ctxb, ref, nil, getUpdateInvoice(htlcID, amt), ) if err != nil { t.Fatalf("unable to settle invoice: %v", err) } // Create the settled invoice for the expectation set. - settleTestInvoice(invoice, settleIndex) + settleTestInvoice(invoice, htlcID, settleIndex) settleIndex++ + htlcID++ } else { pendingInvoices = append(pendingInvoices, *invoice) } @@ -1649,7 +1742,8 @@ func testQueryInvoices(t *testing.T, t.Fatalf("unable to query invoice database: %v", err) } - require.Equal(t, len(testCase.expected), len(response.Invoices)) + require.Equal(t, len(testCase.expected), len(response.Invoices), + fmt.Sprintf("test: #%v", i)) for j, expected := range testCase.expected { require.Equal(t, @@ -1662,7 +1756,9 @@ func testQueryInvoices(t *testing.T, // getUpdateInvoice returns an invoice update callback that, when called, // settles the invoice with the given amount. -func getUpdateInvoice(amt lnwire.MilliSatoshi) invpkg.InvoiceUpdateCallback { +func getUpdateInvoice(htlcID uint64, + amt lnwire.MilliSatoshi) invpkg.InvoiceUpdateCallback { + return func(invoice *invpkg.Invoice) (*invpkg.InvoiceUpdateDesc, error) { @@ -1672,7 +1768,7 @@ func getUpdateInvoice(amt lnwire.MilliSatoshi) invpkg.InvoiceUpdateCallback { noRecords := make(record.CustomSet) htlcs := map[models.CircuitKey]*invpkg.HtlcAcceptDesc{ - {}: { + {HtlcID: htlcID}: { Amt: amt, CustomRecords: noRecords, }, @@ -2082,7 +2178,10 @@ func testSetIDIndex(t *testing.T, makeDB func(t *testing.T) invpkg.InvoiceDB) { ) _, err = db.UpdateInvoice( ctxb, ref2, (*invpkg.SetID)(setID), - updateAcceptAMPHtlc(0, amt, setID, true), + // Note: we need a unique HTLC ID here otherwise the update will + // be rejected as a duplicate (due to SQL unique constraint + // violation). + updateAcceptAMPHtlc(55, amt, setID, true), ) require.Equal(t, invpkg.ErrDuplicateSetID{SetID: *setID}, err) @@ -2366,7 +2465,7 @@ func testUnexpectedInvoicePreimage(t *testing.T, // payment hash against which to validate the preimage. _, err = db.UpdateInvoice( ctxb, invpkg.InvoiceRefByAddr(invoice.Terms.PaymentAddr), nil, - getUpdateInvoice(invoice.Terms.Value), + getUpdateInvoice(0, invoice.Terms.Value), ) // Assert that we get ErrUnexpectedInvoicePreimage. @@ -2501,7 +2600,9 @@ func testDeleteInvoices(t *testing.T, if i == 1 { invoice, err = db.UpdateInvoice( ctxb, invpkg.InvoiceRefByHash(paymentHash), nil, - getUpdateInvoice(invoice.Terms.Value), + getUpdateInvoice( + uint64(i), invoice.Terms.Value, + ), ) require.NoError(t, err, "unable to settle invoice") } diff --git a/invoices/test_utils_test.go b/invoices/test_utils_test.go index dd2638408..a0adf7dc8 100644 --- a/invoices/test_utils_test.go +++ b/invoices/test_utils_test.go @@ -231,7 +231,7 @@ func timeout() func() { go func() { select { - case <-time.After(5 * time.Second): + case <-time.After(10 * time.Second): err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) if err != nil { panic(fmt.Sprintf("error writing to std out "+ diff --git a/itest/lnd_test.go b/itest/lnd_test.go index cb9f60dbf..9c823b5a8 100644 --- a/itest/lnd_test.go +++ b/itest/lnd_test.go @@ -71,6 +71,9 @@ var ( dbBackendFlag = flag.String("dbbackend", "bbolt", "Database backend "+ "(bbolt, etcd, postgres)") + nativeSQLFlag = flag.Bool("nativesql", false, "Database backend to "+ + "use native SQL when applicable (only for sqlite and postgres") + // lndExecutable is the full path to the lnd binary. lndExecutable = flag.String( "lndexec", itestLndBinary, "full path to lnd binary", @@ -95,7 +98,7 @@ func TestLightningNetworkDaemon(t *testing.T) { // Get the binary path and setup the harness test. binary := getLndBinary(t) harnessTest := lntest.SetupHarness( - t, binary, *dbBackendFlag, feeService, + t, binary, *dbBackendFlag, *nativeSQLFlag, feeService, ) defer harnessTest.Stop() diff --git a/lncfg/db.go b/lncfg/db.go index d1ecf6386..49f506bb1 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -3,6 +3,7 @@ package lncfg import ( "context" "fmt" + "path" "path/filepath" "time" @@ -14,6 +15,7 @@ import ( "github.com/lightningnetwork/lnd/kvdb/sqlite" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lnwallet/btcwallet" + "github.com/lightningnetwork/lnd/sqldb" ) const ( @@ -28,6 +30,7 @@ const ( SqliteChainDBName = "chain.sqlite" SqliteNeutrinoDBName = "neutrino.sqlite" SqliteTowerDBName = "watchtower.sqlite" + SqliteNativeDBName = "lnd.sqlite" BoltBackend = "bolt" EtcdBackend = "etcd" @@ -78,9 +81,11 @@ type DB struct { Bolt *kvdb.BoltConfig `group:"bolt" namespace:"bolt" description:"Bolt settings."` - Postgres *postgres.Config `group:"postgres" namespace:"postgres" description:"Postgres settings."` + Postgres *sqldb.PostgresConfig `group:"postgres" namespace:"postgres" description:"Postgres settings."` - Sqlite *sqlite.Config `group:"sqlite" namespace:"sqlite" description:"Sqlite settings."` + Sqlite *sqldb.SqliteConfig `group:"sqlite" namespace:"sqlite" description:"Sqlite settings."` + + UseNativeSQL bool `long:"use-native-sql" description:"Use native SQL for tables that already support it."` NoGraphCache bool `long:"no-graph-cache" description:"Don't use the in-memory graph cache for path finding. Much slower but uses less RAM. Can only be used with a bolt database backend."` @@ -103,13 +108,14 @@ func DefaultDB() *DB { // Allow at most 32 MiB messages by default. MaxMsgSize: 32768 * 1024, }, - Postgres: &postgres.Config{ + Postgres: &sqldb.PostgresConfig{ MaxConnections: defaultPostgresMaxConnections, }, - Sqlite: &sqlite.Config{ + Sqlite: &sqldb.SqliteConfig{ MaxConnections: defaultSqliteMaxConnections, BusyTimeout: defaultSqliteBusyTimeout, }, + UseNativeSQL: false, } } @@ -118,8 +124,8 @@ func (db *DB) Validate() error { switch db.Backend { case BoltBackend, SqliteBackend: case PostgresBackend: - if db.Postgres.Dsn == "" { - return fmt.Errorf("postgres dsn must be set") + if err := db.Postgres.Validate(); err != nil { + return err } case EtcdBackend: @@ -214,6 +220,11 @@ type DatabaseBackends struct { // the underlying wallet database from. WalletDB btcwallet.LoaderOption + // NativeSQLStore is a pointer to a native SQL store that can be used + // for native SQL queries for tables that already support it. This may + // be nil if the use-native-sql flag was not set. + NativeSQLStore *sqldb.BaseDB + // Remote indicates whether the database backends are remote, possibly // replicated instances or local bbolt or sqlite backed databases. Remote bool @@ -223,6 +234,26 @@ type DatabaseBackends struct { CloseFuncs map[string]func() error } +// GetPostgresConfigKVDB converts a sqldb.PostgresConfig to a kvdb +// postgres.Config. +func GetPostgresConfigKVDB(cfg *sqldb.PostgresConfig) *postgres.Config { + return &postgres.Config{ + Dsn: cfg.Dsn, + Timeout: cfg.Timeout, + MaxConnections: cfg.MaxConnections, + } +} + +// GetSqliteConfigKVDB converts a sqldb.SqliteConfig to a kvdb sqlite.Config. +func GetSqliteConfigKVDB(cfg *sqldb.SqliteConfig) *sqlite.Config { + return &sqlite.Config{ + Timeout: cfg.Timeout, + BusyTimeout: cfg.BusyTimeout, + MaxConnections: cfg.MaxConnections, + PragmaOptions: cfg.PragmaOptions, + } +} + // GetBackends returns a set of kvdb.Backends as set in the DB config. func (db *DB) GetBackends(ctx context.Context, chanDBPath, walletDBPath, towerServerDBPath string, towerClientEnabled, @@ -342,9 +373,14 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, }, nil case PostgresBackend: + // Convert the sqldb PostgresConfig to a kvdb postgres.Config. + // This is a temporary measure until we migrate all kvdb SQL + // users to native SQL. + postgresConfig := GetPostgresConfigKVDB(db.Postgres) + postgresBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSChannelDB, + postgresConfig, NSChannelDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres graph "+ @@ -354,7 +390,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, postgresMacaroonBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSMacaroonDB, + postgresConfig, NSMacaroonDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres "+ @@ -364,7 +400,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, postgresDecayedLogBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSDecayedLogDB, + postgresConfig, NSDecayedLogDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres "+ @@ -374,7 +410,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, postgresTowerClientBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSTowerClientDB, + postgresConfig, NSTowerClientDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres tower "+ @@ -384,7 +420,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, postgresTowerServerBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSTowerServerDB, + postgresConfig, NSTowerServerDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres tower "+ @@ -394,7 +430,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, postgresWalletBackend, err := kvdb.Open( kvdb.PostgresBackendName, ctx, - db.Postgres, NSWalletDB, + postgresConfig, NSWalletDB, ) if err != nil { return nil, fmt.Errorf("error opening postgres macaroon "+ @@ -402,6 +438,20 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, } closeFuncs[NSWalletDB] = postgresWalletBackend.Close + var nativeSQLStore *sqldb.BaseDB + if db.UseNativeSQL { + nativePostgresStore, err := sqldb.NewPostgresStore( + db.Postgres, + ) + if err != nil { + return nil, fmt.Errorf("error opening "+ + "native postgres store: %v", err) + } + + nativeSQLStore = nativePostgresStore.BaseDB + closeFuncs[PostgresBackend] = nativePostgresStore.Close + } + // Warn if the user is trying to switch over to a Postgres DB // while there is a wallet or channel bbolt DB still present. warnExistingBoltDBs( @@ -429,11 +479,17 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, WalletDB: btcwallet.LoaderWithExternalWalletDB( postgresWalletBackend, ), - Remote: true, - CloseFuncs: closeFuncs, + NativeSQLStore: nativeSQLStore, + Remote: true, + CloseFuncs: closeFuncs, }, nil case SqliteBackend: + // Convert the sqldb SqliteConfig to a kvdb sqlite.Config. + // This is a temporary measure until we migrate all kvdb SQL + // users to native SQL. + sqliteConfig := GetSqliteConfigKVDB(db.Sqlite) + // Note that for sqlite, we put kv tables for the channel.db, // wtclient.db and sphinxreplay.db all in the channel.sqlite db. // The tables for wallet.db and macaroon.db are in the @@ -445,7 +501,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, // would cause deadlocks in the code due to the wallet db often // being accessed during a write to another db. sqliteBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, chanDBPath, SqliteChannelDBName, NSChannelDB, ) if err != nil { @@ -455,7 +511,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, closeFuncs[NSChannelDB] = sqliteBackend.Close sqliteMacaroonBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, walletDBPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, walletDBPath, SqliteChainDBName, NSMacaroonDB, ) if err != nil { @@ -465,7 +521,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, closeFuncs[NSMacaroonDB] = sqliteMacaroonBackend.Close sqliteDecayedLogBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, chanDBPath, SqliteChannelDBName, NSDecayedLogDB, ) if err != nil { @@ -475,7 +531,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, closeFuncs[NSDecayedLogDB] = sqliteDecayedLogBackend.Close sqliteTowerClientBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, chanDBPath, SqliteChannelDBName, NSTowerClientDB, ) if err != nil { @@ -485,7 +541,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, closeFuncs[NSTowerClientDB] = sqliteTowerClientBackend.Close sqliteTowerServerBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, + kvdb.SqliteBackendName, ctx, sqliteConfig, towerServerDBPath, SqliteTowerDBName, NSTowerServerDB, ) if err != nil { @@ -495,7 +551,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, closeFuncs[NSTowerServerDB] = sqliteTowerServerBackend.Close sqliteWalletBackend, err := kvdb.Open( - kvdb.SqliteBackendName, ctx, db.Sqlite, walletDBPath, + kvdb.SqliteBackendName, ctx, sqliteConfig, walletDBPath, SqliteChainDBName, NSWalletDB, ) if err != nil { @@ -504,6 +560,21 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, } closeFuncs[NSWalletDB] = sqliteWalletBackend.Close + var nativeSQLStore *sqldb.BaseDB + if db.UseNativeSQL { + nativeSQLiteStore, err := sqldb.NewSqliteStore( + db.Sqlite, + path.Join(chanDBPath, SqliteNativeDBName), + ) + if err != nil { + return nil, fmt.Errorf("error opening "+ + "native SQLite store: %v", err) + } + + nativeSQLStore = nativeSQLiteStore.BaseDB + closeFuncs[SqliteBackend] = nativeSQLiteStore.Close + } + // Warn if the user is trying to switch over to a sqlite DB // while there is a wallet or channel bbolt DB still present. warnExistingBoltDBs( @@ -531,7 +602,8 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, WalletDB: btcwallet.LoaderWithExternalWalletDB( sqliteWalletBackend, ), - CloseFuncs: closeFuncs, + NativeSQLStore: nativeSQLStore, + CloseFuncs: closeFuncs, }, nil } diff --git a/lntest/harness.go b/lntest/harness.go index c0a5da7f0..1b9d18fae 100644 --- a/lntest/harness.go +++ b/lntest/harness.go @@ -103,14 +103,14 @@ type HarnessTest struct { // NewHarnessTest creates a new instance of a harnessTest from a regular // testing.T instance. func NewHarnessTest(t *testing.T, lndBinary string, feeService WebFeeService, - dbBackend node.DatabaseBackend) *HarnessTest { + dbBackend node.DatabaseBackend, nativeSQL bool) *HarnessTest { t.Helper() // Create the run context. ctxt, cancel := context.WithCancel(context.Background()) - manager := newNodeManager(lndBinary, dbBackend) + manager := newNodeManager(lndBinary, dbBackend, nativeSQL) return &HarnessTest{ T: t, diff --git a/lntest/harness_node_manager.go b/lntest/harness_node_manager.go index dd1b59ea3..3b0c412d9 100644 --- a/lntest/harness_node_manager.go +++ b/lntest/harness_node_manager.go @@ -31,6 +31,10 @@ type nodeManager struct { // dbBackend sets the database backend to use. dbBackend node.DatabaseBackend + // nativeSQL sets the database backend to use native SQL when + // applicable. + nativeSQL bool + // activeNodes is a map of all running nodes, format: // {pubkey: *HarnessNode}. activeNodes map[uint32]*node.HarnessNode @@ -48,12 +52,13 @@ type nodeManager struct { } // newNodeManager creates a new node manager instance. -func newNodeManager(lndBinary string, - dbBackend node.DatabaseBackend) *nodeManager { +func newNodeManager(lndBinary string, dbBackend node.DatabaseBackend, + nativeSQL bool) *nodeManager { return &nodeManager{ lndBinary: lndBinary, dbBackend: dbBackend, + nativeSQL: nativeSQL, activeNodes: make(map[uint32]*node.HarnessNode), standbyNodes: make(map[uint32]*node.HarnessNode), } @@ -80,6 +85,7 @@ func (nm *nodeManager) newNode(t *testing.T, name string, extraArgs []string, ExtraArgs: extraArgs, FeeURL: nm.feeServiceURL, DBBackend: nm.dbBackend, + NativeSQL: nm.nativeSQL, NodeID: nm.nextNodeID(), LndBinary: nm.lndBinary, NetParams: harnessNetParams, diff --git a/lntest/harness_setup.go b/lntest/harness_setup.go index 20442f9c2..61eac5d2f 100644 --- a/lntest/harness_setup.go +++ b/lntest/harness_setup.go @@ -18,7 +18,7 @@ import ( // 4. connect the miner and the chain backend. // 5. start the HarnessTest. func SetupHarness(t *testing.T, binaryPath, dbBackendName string, - feeService WebFeeService) *HarnessTest { + nativeSQL bool, feeService WebFeeService) *HarnessTest { t.Log("Setting up HarnessTest...") @@ -30,7 +30,7 @@ func SetupHarness(t *testing.T, binaryPath, dbBackendName string, dbBackend := prepareDBBackend(t, dbBackendName) // Create a new HarnessTest. - ht := NewHarnessTest(t, binaryPath, feeService, dbBackend) + ht := NewHarnessTest(t, binaryPath, feeService, dbBackend, nativeSQL) // Init the miner. t.Log("Prepare the miner and mine blocks to activate segwit...") diff --git a/lntest/node/config.go b/lntest/node/config.go index 449ad2fad..1573af42e 100644 --- a/lntest/node/config.go +++ b/lntest/node/config.go @@ -119,6 +119,7 @@ type BaseNodeConfig struct { DBBackend DatabaseBackend PostgresDsn string + NativeSQL bool // NodeID is a unique ID used to identify the node. NodeID uint32 @@ -277,11 +278,17 @@ func (cfg *BaseNodeConfig) GenArgs() []string { case BackendPostgres: args = append(args, "--db.backend=postgres") args = append(args, "--db.postgres.dsn="+cfg.PostgresDsn) + if cfg.NativeSQL { + args = append(args, "--db.use-native-sql") + } case BackendSqlite: args = append(args, "--db.backend=sqlite") args = append(args, fmt.Sprintf("--db.sqlite.busytimeout=%v", wait.SqliteBusyTimeout)) + if cfg.NativeSQL { + args = append(args, "--db.use-native-sql") + } } if cfg.FeeURL != "" { diff --git a/make/testing_flags.mk b/make/testing_flags.mk index 68afc9aa0..df20cdb7e 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -60,10 +60,16 @@ DEV_TAGS += kvdb_etcd endif ifeq ($(dbbackend),postgres) +ifneq ($(nativesql),) +ITEST_FLAGS += -nativesql +endif DEV_TAGS += kvdb_postgres endif ifeq ($(dbbackend),sqlite) +ifneq ($(nativesql),) +ITEST_FLAGS += -nativesql +endif DEV_TAGS += kvdb_sqlite endif diff --git a/rpcserver.go b/rpcserver.go index fdfb8ebd6..d55c97257 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -5806,7 +5806,7 @@ func (r *rpcServer) ListInvoices(ctx context.Context, CreationDateEnd: int64(req.CreationDateEnd), } - invoiceSlice, err := r.server.miscDB.QueryInvoices(ctx, q) + invoiceSlice, err := r.server.invoicesDB.QueryInvoices(ctx, q) if err != nil { return nil, fmt.Errorf("unable to query invoices: %w", err) } diff --git a/sample-lnd.conf b/sample-lnd.conf index cac080c7b..abc0ea80f 100644 --- a/sample-lnd.conf +++ b/sample-lnd.conf @@ -1309,6 +1309,11 @@ ; the future. ; db.no-rev-log-amt-data=false +; If set to true, native SQL will be used instead of KV emulation for tables +; that support it already. Note: this is an experimental feature, use at your +; own risk. +; db.use-native-sql=false + [etcd] @@ -1404,6 +1409,9 @@ ; Example: ; db.postgres.maxconnections= +; Whether to skip executing schema migrations. +; db.postgres.skipmigrations=false + [sqlite] @@ -1428,6 +1436,8 @@ ; db.sqlite.pragmaoptions=auto_vacuum=incremental ; db.sqlite.pragmaoptions=temp_store=MEMORY +; Whether to skip executing schema migrations. +; db.sqlite.skipmigrations=false [bolt] diff --git a/scripts/gen_sqlc_docker.sh b/scripts/gen_sqlc_docker.sh index 9c8e88487..e011d689e 100755 --- a/scripts/gen_sqlc_docker.sh +++ b/scripts/gen_sqlc_docker.sh @@ -16,4 +16,4 @@ docker run \ -e UID=$UID \ -v "$DIR/../:/build" \ -w /build \ - kjconroy/sqlc:1.18.0 generate + sqlc/sqlc:1.25.0 generate diff --git a/server.go b/server.go index cb1540a2c..32dbc9cf0 100644 --- a/server.go +++ b/server.go @@ -246,6 +246,8 @@ type server struct { // channel DB that haven't been separated out yet. miscDB *channeldb.DB + invoicesDB invoices.InvoiceDB + aliasMgr *aliasmgr.Manager htlcSwitch *htlcswitch.Switch @@ -564,6 +566,7 @@ func newServer(cfg *Config, listenAddrs []net.Addr, chanStateDB: dbs.ChanStateDB.ChannelStateDB(), addrSource: dbs.ChanStateDB, miscDB: dbs.ChanStateDB, + invoicesDB: dbs.InvoiceDB, cc: cc, sigPool: lnwallet.NewSigPool(cfg.Workers.Sig, cc.Signer), writePool: writePool, diff --git a/sqldb/config.go b/sqldb/config.go new file mode 100644 index 000000000..5bc807806 --- /dev/null +++ b/sqldb/config.go @@ -0,0 +1,55 @@ +package sqldb + +import ( + "fmt" + "net/url" + "time" +) + +const ( + // defaultMaxConns is the number of permitted active and idle + // connections. We want to limit this so it isn't unlimited. We use the + // same value for the number of idle connections as, this can speed up + // queries given a new connection doesn't need to be established each + // time. + defaultMaxConns = 25 + + // connIdleLifetime is the amount of time a connection can be idle. + connIdleLifetime = 5 * time.Minute +) + +// SqliteConfig holds all the config arguments needed to interact with our +// sqlite DB. +// +//nolint:lll +type SqliteConfig struct { + Timeout time.Duration `long:"timeout" description:"The time after which a database query should be timed out."` + BusyTimeout time.Duration `long:"busytimeout" description:"The maximum amount of time to wait for a database connection to become available for a query."` + MaxConnections int `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."` + PragmaOptions []string `long:"pragmaoptions" description:"A list of pragma options to set on a database connection. For example, 'auto_vacuum=incremental'. Note that the flag must be specified multiple times if multiple options are to be set."` + SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` +} + +// PostgresConfig holds the postgres database configuration. +// +//nolint:lll +type PostgresConfig struct { + Dsn string `long:"dsn" description:"Database connection string."` + Timeout time.Duration `long:"timeout" description:"Database connection timeout. Set to zero to disable."` + MaxConnections int `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."` + SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` +} + +func (p *PostgresConfig) Validate() error { + if p.Dsn == "" { + return fmt.Errorf("DSN is required") + } + + // Parse the DSN as a URL. + _, err := url.Parse(p.Dsn) + if err != nil { + return fmt.Errorf("invalid DSN: %w", err) + } + + return nil +} diff --git a/sqldb/invoices.go b/sqldb/invoices.go new file mode 100644 index 000000000..6e2cb47a1 --- /dev/null +++ b/sqldb/invoices.go @@ -0,0 +1,1658 @@ +package sqldb + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "math" + "strconv" + "time" + + "github.com/lightningnetwork/lnd/channeldb/models" + "github.com/lightningnetwork/lnd/clock" + invpkg "github.com/lightningnetwork/lnd/invoices" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/sqldb/sqlc" +) + +const ( + // queryPaginationLimit is used in the LIMIT clause of the SQL queries + // to limit the number of rows returned. + queryPaginationLimit = 100 +) + +// InvoiceQueries is an interface that defines the set of operations that can be +// executed against the invoice database. +type InvoiceQueries interface { //nolint:interfacebloat + InsertInvoice(ctx context.Context, arg sqlc.InsertInvoiceParams) (int64, + error) + + InsertInvoiceFeature(ctx context.Context, + arg sqlc.InsertInvoiceFeatureParams) error + + InsertInvoiceHTLC(ctx context.Context, + arg sqlc.InsertInvoiceHTLCParams) (int64, error) + + InsertInvoiceHTLCCustomRecord(ctx context.Context, + arg sqlc.InsertInvoiceHTLCCustomRecordParams) error + + FilterInvoices(ctx context.Context, + arg sqlc.FilterInvoicesParams) ([]sqlc.Invoice, error) + + GetInvoice(ctx context.Context, + arg sqlc.GetInvoiceParams) ([]sqlc.Invoice, error) + + GetInvoiceFeatures(ctx context.Context, + invoiceID int64) ([]sqlc.InvoiceFeature, error) + + GetInvoiceHTLCCustomRecords(ctx context.Context, + invoiceID int64) ([]sqlc.GetInvoiceHTLCCustomRecordsRow, error) + + GetInvoiceHTLCs(ctx context.Context, + invoiceID int64) ([]sqlc.InvoiceHtlc, error) + + UpdateInvoiceState(ctx context.Context, + arg sqlc.UpdateInvoiceStateParams) (sql.Result, error) + + UpdateInvoiceAmountPaid(ctx context.Context, + arg sqlc.UpdateInvoiceAmountPaidParams) (sql.Result, error) + + NextInvoiceSettleIndex(ctx context.Context) (int64, error) + + UpdateInvoiceHTLC(ctx context.Context, + arg sqlc.UpdateInvoiceHTLCParams) error + + DeleteInvoice(ctx context.Context, arg sqlc.DeleteInvoiceParams) ( + sql.Result, error) + + DeleteCanceledInvoices(ctx context.Context) (sql.Result, error) + + // AMP sub invoice specific methods. + UpsertAMPSubInvoice(ctx context.Context, + arg sqlc.UpsertAMPSubInvoiceParams) (sql.Result, error) + + UpdateAMPSubInvoiceState(ctx context.Context, + arg sqlc.UpdateAMPSubInvoiceStateParams) error + + InsertAMPSubInvoiceHTLC(ctx context.Context, + arg sqlc.InsertAMPSubInvoiceHTLCParams) error + + FetchAMPSubInvoices(ctx context.Context, + arg sqlc.FetchAMPSubInvoicesParams) ([]sqlc.AmpSubInvoice, + error) + + FetchAMPSubInvoiceHTLCs(ctx context.Context, + arg sqlc.FetchAMPSubInvoiceHTLCsParams) ( + []sqlc.FetchAMPSubInvoiceHTLCsRow, error) + + FetchSettledAMPSubInvoices(ctx context.Context, + arg sqlc.FetchSettledAMPSubInvoicesParams) ( + []sqlc.FetchSettledAMPSubInvoicesRow, error) + + UpdateAMPSubInvoiceHTLCPreimage(ctx context.Context, + arg sqlc.UpdateAMPSubInvoiceHTLCPreimageParams) (sql.Result, + error) + + // Invoice events specific methods. + OnInvoiceCreated(ctx context.Context, + arg sqlc.OnInvoiceCreatedParams) error + + OnInvoiceCanceled(ctx context.Context, + arg sqlc.OnInvoiceCanceledParams) error + + OnInvoiceSettled(ctx context.Context, + arg sqlc.OnInvoiceSettledParams) error + + OnAMPSubInvoiceCreated(ctx context.Context, + arg sqlc.OnAMPSubInvoiceCreatedParams) error + + OnAMPSubInvoiceCanceled(ctx context.Context, + arg sqlc.OnAMPSubInvoiceCanceledParams) error + + OnAMPSubInvoiceSettled(ctx context.Context, + arg sqlc.OnAMPSubInvoiceSettledParams) error +} + +var _ invpkg.InvoiceDB = (*InvoiceStore)(nil) + +// InvoiceQueriesTxOptions defines the set of db txn options the InvoiceQueries +// understands. +type InvoiceQueriesTxOptions struct { + // readOnly governs if a read only transaction is needed or not. + readOnly bool +} + +// ReadOnly returns true if the transaction should be read only. +// +// NOTE: This implements the TxOptions. +func (a *InvoiceQueriesTxOptions) ReadOnly() bool { + return a.readOnly +} + +// NewInvoiceQueryReadTx creates a new read transaction option set. +func NewInvoiceQueryReadTx() InvoiceQueriesTxOptions { + return InvoiceQueriesTxOptions{ + readOnly: true, + } +} + +// BatchedInvoiceQueries is a version of the InvoiceQueries that's capable of +// batched database operations. +type BatchedInvoiceQueries interface { + InvoiceQueries + + BatchedTx[InvoiceQueries] +} + +// InvoiceStore represents a storage backend. +type InvoiceStore struct { + db BatchedInvoiceQueries + clock clock.Clock +} + +// NewInvoiceStore creates a new InvoiceStore instance given a open +// BatchedInvoiceQueries storage backend. +func NewInvoiceStore(db BatchedInvoiceQueries, + clock clock.Clock) *InvoiceStore { + + return &InvoiceStore{ + db: db, + clock: clock, + } +} + +// AddInvoice inserts the targeted invoice into the database. If the invoice has +// *any* payment hashes which already exists within the database, then the +// insertion will be aborted and rejected due to the strict policy banning any +// duplicate payment hashes. +// +// NOTE: A side effect of this function is that it sets AddIndex on newInvoice. +func (i *InvoiceStore) AddInvoice(ctx context.Context, + newInvoice *invpkg.Invoice, paymentHash lntypes.Hash) (uint64, error) { + + // Make sure this is a valid invoice before trying to store it in our + // DB. + if err := invpkg.ValidateInvoice(newInvoice, paymentHash); err != nil { + return 0, err + } + + var ( + writeTxOpts InvoiceQueriesTxOptions + invoiceID int64 + ) + + // Precompute the payment request hash so we can use it in the query. + var paymentRequestHash []byte + if len(newInvoice.PaymentRequest) > 0 { + h := sha256.New() + h.Write(newInvoice.PaymentRequest) + paymentRequestHash = h.Sum(nil) + } + + err := i.db.ExecTx(ctx, &writeTxOpts, func(db InvoiceQueries) error { + params := sqlc.InsertInvoiceParams{ + Hash: paymentHash[:], + Memo: sqlStr(string(newInvoice.Memo)), + AmountMsat: int64(newInvoice.Terms.Value), + // Note: BOLT12 invoices don't have a final cltv delta. + CltvDelta: sqlInt32(newInvoice.Terms.FinalCltvDelta), + Expiry: int32(newInvoice.Terms.Expiry), + // Note: keysend invoices don't have a payment request. + PaymentRequest: sqlStr(string( + newInvoice.PaymentRequest), + ), + PaymentRequestHash: paymentRequestHash, + State: int16(newInvoice.State), + AmountPaidMsat: int64(newInvoice.AmtPaid), + IsAmp: newInvoice.IsAMP(), + IsHodl: newInvoice.HodlInvoice, + IsKeysend: newInvoice.IsKeysend(), + CreatedAt: newInvoice.CreationDate.UTC(), + } + + // Some invoices may not have a preimage, like in the case of + // HODL invoices. + if newInvoice.Terms.PaymentPreimage != nil { + preimage := *newInvoice.Terms.PaymentPreimage + if preimage == invpkg.UnknownPreimage { + return errors.New("cannot use all-zeroes " + + "preimage") + } + params.Preimage = preimage[:] + } + + // Some non MPP payments may have the default (invalid) value. + if newInvoice.Terms.PaymentAddr != invpkg.BlankPayAddr { + params.PaymentAddr = newInvoice.Terms.PaymentAddr[:] + } + + var err error + invoiceID, err = db.InsertInvoice(ctx, params) + if err != nil { + return fmt.Errorf("unable to insert invoice: %w", err) + } + + // TODO(positiveblue): if invocies do not have custom features + // maybe just store the "invoice type" and populate the features + // based on that. + for feature := range newInvoice.Terms.Features.Features() { + params := sqlc.InsertInvoiceFeatureParams{ + InvoiceID: invoiceID, + Feature: int32(feature), + } + + err := db.InsertInvoiceFeature(ctx, params) + if err != nil { + return fmt.Errorf("unable to insert invoice "+ + "feature(%v): %w", feature, err) + } + } + + // Finally add a new event for this invoice. + return db.OnInvoiceCreated(ctx, sqlc.OnInvoiceCreatedParams{ + AddedAt: newInvoice.CreationDate.UTC(), + InvoiceID: invoiceID, + }) + }) + if err != nil { + mappedSQLErr := MapSQLError(err) + var uniqueConstraintErr *ErrSQLUniqueConstraintViolation + if errors.As(mappedSQLErr, &uniqueConstraintErr) { + // Add context to unique constraint errors. + return 0, invpkg.ErrDuplicateInvoice + } + + return 0, fmt.Errorf("unable to add invoice(%v): %w", + paymentHash, err) + } + + newInvoice.AddIndex = uint64(invoiceID) + + return newInvoice.AddIndex, nil +} + +// fetchInvoice fetches the common invoice data and the AMP state for the +// invoice with the given reference. +func (i *InvoiceStore) fetchInvoice(ctx context.Context, + db InvoiceQueries, ref invpkg.InvoiceRef) (*invpkg.Invoice, error) { + + if ref.PayHash() == nil && ref.PayAddr() == nil && ref.SetID() == nil { + return nil, invpkg.ErrInvoiceNotFound + } + + var ( + invoice *invpkg.Invoice + params sqlc.GetInvoiceParams + ) + + // Given all invoices are uniquely identified by their payment hash, + // we can use it to query a specific invoice. + if ref.PayHash() != nil { + params.Hash = ref.PayHash()[:] + } + + // Newer invoices (0.11 and up) are indexed by payment address in + // addition to payment hash, but pre 0.8 invoices do not have one at + // all. Only allow lookups for payment address if it is not a blank + // payment address, which is a special-cased value for legacy keysend + // invoices. + if ref.PayAddr() != nil && *ref.PayAddr() != invpkg.BlankPayAddr { + params.PaymentAddr = ref.PayAddr()[:] + } + + // If the reference has a set ID we'll fetch the invoice which has the + // corresponding AMP sub invoice. + if ref.SetID() != nil { + params.SetID = ref.SetID()[:] + } + + rows, err := db.GetInvoice(ctx, params) + switch { + case len(rows) == 0: + return nil, invpkg.ErrInvoiceNotFound + + case len(rows) > 1: + // In case the reference is ambiguous, meaning it matches more + // than one invoice, we'll return an error. + return nil, fmt.Errorf("ambiguous invoice ref: %s", + ref.String()) + + case err != nil: + return nil, fmt.Errorf("unable to fetch invoice: %w", err) + } + + var ( + setID *[32]byte + fetchAmpHtlcs bool + ) + + // Now that we got the invoice itself, fetch the HTLCs as requested by + // the modifier. + switch ref.Modifier() { + case invpkg.DefaultModifier: + // By default we'll fetch all AMP HTLCs. + setID = nil + fetchAmpHtlcs = true + + case invpkg.HtlcSetOnlyModifier: + // In this case we'll fetch all AMP HTLCs for the + // specified set id. + if ref.SetID() == nil { + return nil, fmt.Errorf("set ID is required to use " + + "the HTLC set only modifier") + } + + setID = ref.SetID() + fetchAmpHtlcs = true + + case invpkg.HtlcSetBlankModifier: + // No need to fetch any HTLCs. + setID = nil + fetchAmpHtlcs = false + + default: + return nil, fmt.Errorf("unknown invoice ref modifier: %v", + ref.Modifier()) + } + + // Fetch the rest of the invoice data and fill the invoice struct. + _, invoice, err = fetchInvoiceData( + ctx, db, rows[0], setID, fetchAmpHtlcs, + ) + if err != nil { + return nil, err + } + + return invoice, nil +} + +// fetchAmpState fetches the AMP state for the invoice with the given ID. +// Optional setID can be provided to fetch the state for a specific AMP HTLC +// set. If setID is nil then we'll fetch the state for all AMP sub invoices. If +// fetchHtlcs is set to true, the HTLCs for the given set will be fetched as +// well. +// +//nolint:funlen +func fetchAmpState(ctx context.Context, db InvoiceQueries, invoiceID int64, + setID *[32]byte, fetchHtlcs bool) (invpkg.AMPInvoiceState, + invpkg.HTLCSet, error) { + + var paramSetID []byte + if setID != nil { + paramSetID = setID[:] + } + + // First fetch all the AMP sub invoices for this invoice or the one + // matching the provided set ID. + ampInvoiceRows, err := db.FetchAMPSubInvoices( + ctx, sqlc.FetchAMPSubInvoicesParams{ + InvoiceID: invoiceID, + SetID: paramSetID, + }, + ) + if err != nil { + return nil, nil, err + } + + ampState := make(map[invpkg.SetID]invpkg.InvoiceStateAMP) + for _, row := range ampInvoiceRows { + var rowSetID [32]byte + + if len(row.SetID) != 32 { + return nil, nil, fmt.Errorf("invalid set id length: %d", + len(row.SetID)) + } + + var settleDate time.Time + if row.SettledAt.Valid { + settleDate = row.SettledAt.Time.Local() + } + + copy(rowSetID[:], row.SetID) + ampState[rowSetID] = invpkg.InvoiceStateAMP{ + State: invpkg.HtlcState(row.State), + SettleIndex: uint64(row.SettleIndex.Int64), + SettleDate: settleDate, + InvoiceKeys: make(map[models.CircuitKey]struct{}), + } + } + + if !fetchHtlcs { + return ampState, nil, nil + } + + customRecordRows, err := db.GetInvoiceHTLCCustomRecords(ctx, invoiceID) + if err != nil { + return nil, nil, fmt.Errorf("unable to get custom records for "+ + "invoice HTLCs: %w", err) + } + + customRecords := make(map[int64]record.CustomSet, len(customRecordRows)) + for _, row := range customRecordRows { + if _, ok := customRecords[row.HtlcID]; !ok { + customRecords[row.HtlcID] = make(record.CustomSet) + } + + value := row.Value + if value == nil { + value = []byte{} + } + + customRecords[row.HtlcID][uint64(row.Key)] = value + } + + // Now fetch all the AMP HTLCs for this invoice or the one matching the + // provided set ID. + ampHtlcRows, err := db.FetchAMPSubInvoiceHTLCs( + ctx, sqlc.FetchAMPSubInvoiceHTLCsParams{ + InvoiceID: invoiceID, + SetID: paramSetID, + }, + ) + if err != nil { + return nil, nil, err + } + + ampHtlcs := make(map[models.CircuitKey]*invpkg.InvoiceHTLC) + for _, row := range ampHtlcRows { + uint64ChanID, err := strconv.ParseUint(row.ChanID, 10, 64) + if err != nil { + return nil, nil, err + } + + chanID := lnwire.NewShortChanIDFromInt(uint64ChanID) + + if row.HtlcID < 0 { + return nil, nil, fmt.Errorf("invalid HTLC ID "+ + "value: %v", row.HtlcID) + } + + htlcID := uint64(row.HtlcID) + + circuitKey := invpkg.CircuitKey{ + ChanID: chanID, + HtlcID: htlcID, + } + + htlc := &invpkg.InvoiceHTLC{ + Amt: lnwire.MilliSatoshi(row.AmountMsat), + AcceptHeight: uint32(row.AcceptHeight), + AcceptTime: row.AcceptTime.Local(), + Expiry: uint32(row.ExpiryHeight), + State: invpkg.HtlcState(row.State), + } + + if row.TotalMppMsat.Valid { + htlc.MppTotalAmt = lnwire.MilliSatoshi( + row.TotalMppMsat.Int64, + ) + } + + if row.ResolveTime.Valid { + htlc.ResolveTime = row.ResolveTime.Time.Local() + } + + var ( + rootShare [32]byte + setID [32]byte + ) + + if len(row.RootShare) != 32 { + return nil, nil, fmt.Errorf("invalid root share "+ + "length: %d", len(row.RootShare)) + } + copy(rootShare[:], row.RootShare) + + if len(row.SetID) != 32 { + return nil, nil, fmt.Errorf("invalid set ID length: %d", + len(row.SetID)) + } + copy(setID[:], row.SetID) + + if row.ChildIndex < 0 || row.ChildIndex > math.MaxUint32 { + return nil, nil, fmt.Errorf("invalid child index "+ + "value: %v", row.ChildIndex) + } + + ampRecord := record.NewAMP( + rootShare, setID, uint32(row.ChildIndex), + ) + + htlc.AMP = &invpkg.InvoiceHtlcAMPData{ + Record: *ampRecord, + } + + if len(row.Hash) != 32 { + return nil, nil, fmt.Errorf("invalid hash length: %d", + len(row.Hash)) + } + copy(htlc.AMP.Hash[:], row.Hash) + + if row.Preimage != nil { + preimage, err := lntypes.MakePreimage(row.Preimage) + if err != nil { + return nil, nil, err + } + + htlc.AMP.Preimage = &preimage + } + + if _, ok := customRecords[row.ID]; ok { + htlc.CustomRecords = customRecords[row.ID] + } else { + htlc.CustomRecords = make(record.CustomSet) + } + + ampHtlcs[circuitKey] = htlc + } + + if len(ampHtlcs) > 0 { + for setID := range ampState { + var amtPaid lnwire.MilliSatoshi + invoiceKeys := make( + map[models.CircuitKey]struct{}, + ) + + for key, htlc := range ampHtlcs { + if htlc.AMP.Record.SetID() != setID { + continue + } + + invoiceKeys[key] = struct{}{} + + if htlc.State != invpkg.HtlcStateCanceled { //nolint: lll + amtPaid += htlc.Amt + } + } + + setState := ampState[setID] + setState.InvoiceKeys = invoiceKeys + setState.AmtPaid = amtPaid + ampState[setID] = setState + } + } + + return ampState, ampHtlcs, nil +} + +// LookupInvoice attempts to look up an invoice corresponding the passed in +// reference. The reference may be a payment hash, a payment address, or a set +// ID for an AMP sub invoice. If the invoice is found, we'll return the complete +// invoice. If the invoice is not found, then we'll return an ErrInvoiceNotFound +// error. +func (i *InvoiceStore) LookupInvoice(ctx context.Context, + ref invpkg.InvoiceRef) (invpkg.Invoice, error) { + + var ( + invoice *invpkg.Invoice + err error + ) + + readTxOpt := NewInvoiceQueryReadTx() + txErr := i.db.ExecTx(ctx, &readTxOpt, func(db InvoiceQueries) error { + invoice, err = i.fetchInvoice(ctx, db, ref) + + return err + }) + if txErr != nil { + return invpkg.Invoice{}, txErr + } + + return *invoice, nil +} + +// FetchPendingInvoices returns all the invoices that are currently in a +// "pending" state. An invoice is pending if it has been created but not yet +// settled or canceled. +func (i *InvoiceStore) FetchPendingInvoices(ctx context.Context) ( + map[lntypes.Hash]invpkg.Invoice, error) { + + invoices := make(map[lntypes.Hash]invpkg.Invoice) + + readTxOpt := NewInvoiceQueryReadTx() + err := i.db.ExecTx(ctx, &readTxOpt, func(db InvoiceQueries) error { + limit := queryPaginationLimit + + return queryWithLimit(func(offset int) (int, error) { + params := sqlc.FilterInvoicesParams{ + PendingOnly: true, + NumOffset: int32(offset), + NumLimit: int32(limit), + } + + rows, err := db.FilterInvoices(ctx, params) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("unable to get invoices "+ + "from db: %w", err) + } + + // Load all the information for the invoices. + for _, row := range rows { + hash, invoice, err := fetchInvoiceData( + ctx, db, row, nil, true, + ) + if err != nil { + return 0, err + } + + invoices[*hash] = *invoice + } + + return len(rows), nil + }, limit) + }) + if err != nil { + return nil, fmt.Errorf("unable to fetch pending invoices: %w", + err) + } + + return invoices, nil +} + +// InvoicesSettledSince can be used by callers to catch up any settled invoices +// they missed within the settled invoice time series. We'll return all known +// settled invoice that have a settle index higher than the passed +// sinceSettleIndex. +// +// NOTE: The index starts from 1. As a result we enforce that specifying a value +// below the starting index value is a noop. +func (i *InvoiceStore) InvoicesSettledSince(ctx context.Context, idx uint64) ( + []invpkg.Invoice, error) { + + var invoices []invpkg.Invoice + + if idx == 0 { + return invoices, nil + } + + readTxOpt := NewInvoiceQueryReadTx() + err := i.db.ExecTx(ctx, &readTxOpt, func(db InvoiceQueries) error { + settleIdx := idx + limit := queryPaginationLimit + + err := queryWithLimit(func(offset int) (int, error) { + params := sqlc.FilterInvoicesParams{ + SettleIndexGet: sqlInt64(settleIdx + 1), + NumLimit: int32(limit), + NumOffset: int32(offset), + } + + rows, err := db.FilterInvoices(ctx, params) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("unable to get invoices "+ + "from db: %w", err) + } + + // Load all the information for the invoices. + for _, row := range rows { + _, invoice, err := fetchInvoiceData( + ctx, db, row, nil, true, + ) + if err != nil { + return 0, fmt.Errorf("unable to fetch "+ + "invoice(id=%d) from db: %w", + row.ID, err) + } + + invoices = append(invoices, *invoice) + } + + settleIdx += uint64(limit) + + return len(rows), nil + }, limit) + if err != nil { + return err + } + + // Now fetch all the AMP sub invoices that were settled since + // the provided index. + ampInvoices, err := i.db.FetchSettledAMPSubInvoices( + ctx, sqlc.FetchSettledAMPSubInvoicesParams{ + SettleIndexGet: sqlInt64(idx + 1), + }, + ) + if err != nil { + return err + } + + for _, ampInvoice := range ampInvoices { + // Convert the row to a sqlc.Invoice so we can use the + // existing fetchInvoiceData function. + sqlInvoice := sqlc.Invoice{ + ID: ampInvoice.ID, + Hash: ampInvoice.Hash, + Preimage: ampInvoice.Preimage, + SettleIndex: ampInvoice.AmpSettleIndex, + SettledAt: ampInvoice.AmpSettledAt, + Memo: ampInvoice.Memo, + AmountMsat: ampInvoice.AmountMsat, + CltvDelta: ampInvoice.CltvDelta, + Expiry: ampInvoice.Expiry, + PaymentAddr: ampInvoice.PaymentAddr, + PaymentRequest: ampInvoice.PaymentRequest, + State: ampInvoice.State, + AmountPaidMsat: ampInvoice.AmountPaidMsat, + IsAmp: ampInvoice.IsAmp, + IsHodl: ampInvoice.IsHodl, + IsKeysend: ampInvoice.IsKeysend, + CreatedAt: ampInvoice.CreatedAt.UTC(), + } + + // Fetch the state and HTLCs for this AMP sub invoice. + _, invoice, err := fetchInvoiceData( + ctx, db, sqlInvoice, + (*[32]byte)(ampInvoice.SetID), true, + ) + if err != nil { + return fmt.Errorf("unable to fetch "+ + "AMP invoice(id=%d) from db: %w", + ampInvoice.ID, err) + } + + invoices = append(invoices, *invoice) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("unable to get invoices settled since "+ + "index (excluding) %d: %w", idx, err) + } + + return invoices, nil +} + +// InvoicesAddedSince can be used by callers to seek into the event time series +// of all the invoices added in the database. This method will return all +// invoices with an add index greater than the specified sinceAddIndex. +// +// NOTE: The index starts from 1. As a result we enforce that specifying a value +// below the starting index value is a noop. +func (i *InvoiceStore) InvoicesAddedSince(ctx context.Context, idx uint64) ( + []invpkg.Invoice, error) { + + var result []invpkg.Invoice + + if idx == 0 { + return result, nil + } + + readTxOpt := NewInvoiceQueryReadTx() + err := i.db.ExecTx(ctx, &readTxOpt, func(db InvoiceQueries) error { + addIdx := idx + limit := queryPaginationLimit + + return queryWithLimit(func(offset int) (int, error) { + params := sqlc.FilterInvoicesParams{ + AddIndexGet: sqlInt64(addIdx + 1), + NumLimit: int32(limit), + NumOffset: int32(offset), + } + + rows, err := db.FilterInvoices(ctx, params) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("unable to get invoices "+ + "from db: %w", err) + } + + // Load all the information for the invoices. + for _, row := range rows { + _, invoice, err := fetchInvoiceData( + ctx, db, row, nil, true, + ) + if err != nil { + return 0, err + } + + result = append(result, *invoice) + } + + addIdx += uint64(limit) + + return len(rows), nil + }, limit) + }) + + if err != nil { + return nil, fmt.Errorf("unable to get invoices added since "+ + "index %d: %w", idx, err) + } + + return result, nil +} + +// QueryInvoices allows a caller to query the invoice database for invoices +// within the specified add index range. +func (i *InvoiceStore) QueryInvoices(ctx context.Context, + q invpkg.InvoiceQuery) (invpkg.InvoiceSlice, error) { + + var invoices []invpkg.Invoice + + if q.NumMaxInvoices == 0 { + return invpkg.InvoiceSlice{}, fmt.Errorf("max invoices must " + + "be non-zero") + } + + readTxOpt := NewInvoiceQueryReadTx() + err := i.db.ExecTx(ctx, &readTxOpt, func(db InvoiceQueries) error { + limit := queryPaginationLimit + + return queryWithLimit(func(offset int) (int, error) { + params := sqlc.FilterInvoicesParams{ + NumOffset: int32(offset), + NumLimit: int32(limit), + PendingOnly: q.PendingOnly, + } + + if !q.Reversed { + // The invoice with index offset id must not be + // included in the results. + params.AddIndexGet = sqlInt64( + q.IndexOffset + uint64(offset) + 1, + ) + } + + if q.Reversed { + idx := int32(q.IndexOffset) + + // If the index offset was not set, we want to + // fetch from the lastest invoice. + if idx == 0 { + params.AddIndexLet = sqlInt64( + int64(math.MaxInt64), + ) + } else { + // The invoice with index offset id must + // not be included in the results. + params.AddIndexLet = sqlInt64( + idx - int32(offset) - 1, + ) + } + + params.Reverse = true + } + + if q.CreationDateStart != 0 { + params.CreatedAfter = sqlTime( + time.Unix(q.CreationDateStart, 0).UTC(), + ) + } + + if q.CreationDateEnd != 0 { + params.CreatedBefore = sqlTime( + time.Unix(q.CreationDateEnd, 0).UTC(), + ) + } + + rows, err := db.FilterInvoices(ctx, params) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("unable to get invoices "+ + "from db: %w", err) + } + + // Load all the information for the invoices. + for _, row := range rows { + _, invoice, err := fetchInvoiceData( + ctx, db, row, nil, true, + ) + if err != nil { + return 0, err + } + + invoices = append(invoices, *invoice) + + if len(invoices) == int(q.NumMaxInvoices) { + return 0, nil + } + } + + return len(rows), nil + }, limit) + }) + if err != nil { + return invpkg.InvoiceSlice{}, fmt.Errorf("unable to query "+ + "invoices: %w", err) + } + + if len(invoices) == 0 { + return invpkg.InvoiceSlice{ + InvoiceQuery: q, + }, nil + } + + // If we iterated through the add index in reverse order, then + // we'll need to reverse the slice of invoices to return them in + // forward order. + if q.Reversed { + numInvoices := len(invoices) + for i := 0; i < numInvoices/2; i++ { + reverse := numInvoices - i - 1 + invoices[i], invoices[reverse] = + invoices[reverse], invoices[i] + } + } + + res := invpkg.InvoiceSlice{ + InvoiceQuery: q, + Invoices: invoices, + FirstIndexOffset: invoices[0].AddIndex, + LastIndexOffset: invoices[len(invoices)-1].AddIndex, + } + + return res, nil +} + +// sqlInvoiceUpdater is the implementation of the InvoiceUpdater interface using +// a SQL database as the backend. +type sqlInvoiceUpdater struct { + db InvoiceQueries + ctx context.Context //nolint:containedctx + invoice *invpkg.Invoice + updateTime time.Time +} + +// AddHtlc adds a new htlc to the invoice. +func (s *sqlInvoiceUpdater) AddHtlc(circuitKey models.CircuitKey, + newHtlc *invpkg.InvoiceHTLC) error { + + htlcPrimaryKeyID, err := s.db.InsertInvoiceHTLC( + s.ctx, sqlc.InsertInvoiceHTLCParams{ + HtlcID: int64(circuitKey.HtlcID), + ChanID: strconv.FormatUint( + circuitKey.ChanID.ToUint64(), 10, + ), + AmountMsat: int64(newHtlc.Amt), + TotalMppMsat: sql.NullInt64{ + Int64: int64(newHtlc.MppTotalAmt), + Valid: newHtlc.MppTotalAmt != 0, + }, + AcceptHeight: int32(newHtlc.AcceptHeight), + AcceptTime: newHtlc.AcceptTime.UTC(), + ExpiryHeight: int32(newHtlc.Expiry), + State: int16(newHtlc.State), + InvoiceID: int64(s.invoice.AddIndex), + }, + ) + if err != nil { + return err + } + + for key, value := range newHtlc.CustomRecords { + err = s.db.InsertInvoiceHTLCCustomRecord( + s.ctx, sqlc.InsertInvoiceHTLCCustomRecordParams{ + // TODO(bhandras): schema might be wrong here + // as the custom record key is an uint64. + Key: int64(key), + Value: value, + HtlcID: htlcPrimaryKeyID, + }, + ) + if err != nil { + return err + } + } + + if newHtlc.AMP != nil { + setID := newHtlc.AMP.Record.SetID() + + upsertResult, err := s.db.UpsertAMPSubInvoice( + s.ctx, sqlc.UpsertAMPSubInvoiceParams{ + SetID: setID[:], + CreatedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + }, + ) + if err != nil { + mappedSQLErr := MapSQLError(err) + var uniqueConstraintErr *ErrSQLUniqueConstraintViolation + if errors.As(mappedSQLErr, &uniqueConstraintErr) { + return invpkg.ErrDuplicateSetID{ + SetID: setID, + } + } + + return err + } + + // If we're just inserting the AMP invoice, we'll get a non + // zero rows affected count. + rowsAffected, err := upsertResult.RowsAffected() + if err != nil { + return err + } + if rowsAffected != 0 { + // If we're inserting a new AMP invoice, we'll also + // insert a new invoice event. + err = s.db.OnAMPSubInvoiceCreated( + s.ctx, sqlc.OnAMPSubInvoiceCreatedParams{ + AddedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + SetID: setID[:], + }, + ) + if err != nil { + return err + } + } + + rootShare := newHtlc.AMP.Record.RootShare() + + ampHtlcParams := sqlc.InsertAMPSubInvoiceHTLCParams{ + InvoiceID: int64(s.invoice.AddIndex), + SetID: setID[:], + HtlcID: htlcPrimaryKeyID, + RootShare: rootShare[:], + ChildIndex: int64( + newHtlc.AMP.Record.ChildIndex(), + ), + Hash: newHtlc.AMP.Hash[:], + } + + if newHtlc.AMP.Preimage != nil { + ampHtlcParams.Preimage = newHtlc.AMP.Preimage[:] + } + + err = s.db.InsertAMPSubInvoiceHTLC(s.ctx, ampHtlcParams) + if err != nil { + return err + } + } + + return nil +} + +// ResolveHtlc marks an htlc as resolved with the given state. +func (s *sqlInvoiceUpdater) ResolveHtlc(circuitKey models.CircuitKey, + state invpkg.HtlcState, resolveTime time.Time) error { + + return s.db.UpdateInvoiceHTLC(s.ctx, sqlc.UpdateInvoiceHTLCParams{ + HtlcID: int64(circuitKey.HtlcID), + ChanID: strconv.FormatUint( + circuitKey.ChanID.ToUint64(), 10, + ), + InvoiceID: int64(s.invoice.AddIndex), + State: int16(state), + ResolveTime: sqlTime(resolveTime.UTC()), + }) +} + +// AddAmpHtlcPreimage adds a preimage of an AMP htlc to the AMP sub invoice +// identified by the setID. +func (s *sqlInvoiceUpdater) AddAmpHtlcPreimage(setID [32]byte, + circuitKey models.CircuitKey, preimage lntypes.Preimage) error { + + result, err := s.db.UpdateAMPSubInvoiceHTLCPreimage( + s.ctx, sqlc.UpdateAMPSubInvoiceHTLCPreimageParams{ + InvoiceID: int64(s.invoice.AddIndex), + SetID: setID[:], + HtlcID: int64(circuitKey.HtlcID), + Preimage: preimage[:], + }, + ) + if err != nil { + return err + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return err + } + if rowsAffected == 0 { + return invpkg.ErrInvoiceNotFound + } + + return nil +} + +// UpdateInvoiceState updates the invoice state to the new state. +func (s *sqlInvoiceUpdater) UpdateInvoiceState( + newState invpkg.ContractState, preimage *lntypes.Preimage) error { + + var ( + settleIndex sql.NullInt64 + settledAt sql.NullTime + ) + + switch newState { + case invpkg.ContractSettled: + nextSettleIndex, err := s.db.NextInvoiceSettleIndex(s.ctx) + if err != nil { + return err + } + + settleIndex = sqlInt64(nextSettleIndex) + + // If the invoice is settled, we'll also update the settle time. + settledAt = sqlTime(s.updateTime.UTC()) + + err = s.db.OnInvoiceSettled( + s.ctx, sqlc.OnInvoiceSettledParams{ + AddedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + }, + ) + if err != nil { + return err + } + + case invpkg.ContractCanceled: + err := s.db.OnInvoiceCanceled( + s.ctx, sqlc.OnInvoiceCanceledParams{ + AddedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + }, + ) + if err != nil { + return err + } + } + + params := sqlc.UpdateInvoiceStateParams{ + ID: int64(s.invoice.AddIndex), + State: int16(newState), + SettleIndex: settleIndex, + SettledAt: settledAt, + } + + if preimage != nil { + params.Preimage = preimage[:] + } + + result, err := s.db.UpdateInvoiceState(s.ctx, params) + if err != nil { + return err + } + rowsAffected, err := result.RowsAffected() + if err != nil { + return err + } + + if rowsAffected == 0 { + return invpkg.ErrInvoiceNotFound + } + + if settleIndex.Valid { + s.invoice.SettleIndex = uint64(settleIndex.Int64) + s.invoice.SettleDate = s.updateTime + } + + return nil +} + +// UpdateInvoiceAmtPaid updates the invoice amount paid to the new amount. +func (s *sqlInvoiceUpdater) UpdateInvoiceAmtPaid( + amtPaid lnwire.MilliSatoshi) error { + + _, err := s.db.UpdateInvoiceAmountPaid( + s.ctx, sqlc.UpdateInvoiceAmountPaidParams{ + ID: int64(s.invoice.AddIndex), + AmountPaidMsat: int64(amtPaid), + }, + ) + + return err +} + +// UpdateAmpState updates the state of the AMP sub invoice identified by the +// setID. +func (s *sqlInvoiceUpdater) UpdateAmpState(setID [32]byte, + newState invpkg.InvoiceStateAMP, _ models.CircuitKey) error { + + var ( + settleIndex sql.NullInt64 + settledAt sql.NullTime + ) + + switch newState.State { + case invpkg.HtlcStateSettled: + nextSettleIndex, err := s.db.NextInvoiceSettleIndex(s.ctx) + if err != nil { + return err + } + + settleIndex = sqlInt64(nextSettleIndex) + + // If the invoice is settled, we'll also update the settle time. + settledAt = sqlTime(s.updateTime.UTC()) + + err = s.db.OnAMPSubInvoiceSettled( + s.ctx, sqlc.OnAMPSubInvoiceSettledParams{ + AddedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + SetID: setID[:], + }, + ) + if err != nil { + return err + } + + case invpkg.HtlcStateCanceled: + err := s.db.OnAMPSubInvoiceCanceled( + s.ctx, sqlc.OnAMPSubInvoiceCanceledParams{ + AddedAt: s.updateTime.UTC(), + InvoiceID: int64(s.invoice.AddIndex), + SetID: setID[:], + }, + ) + if err != nil { + return err + } + } + + err := s.db.UpdateAMPSubInvoiceState( + s.ctx, sqlc.UpdateAMPSubInvoiceStateParams{ + SetID: setID[:], + State: int16(newState.State), + SettleIndex: settleIndex, + SettledAt: settledAt, + }, + ) + if err != nil { + return err + } + + return nil +} + +// Finalize finalizes the update before it is written to the database. Note that +// we don't use this directly in the SQL implementation, so the function is just +// a stub. +func (s *sqlInvoiceUpdater) Finalize(_ invpkg.UpdateType) error { + return nil +} + +// UpdateInvoice attempts to update an invoice corresponding to the passed +// reference. If an invoice matching the passed reference doesn't exist within +// the database, then the action will fail with ErrInvoiceNotFound error. +// +// The update is performed inside the same database transaction that fetches the +// invoice and is therefore atomic. The fields to update are controlled by the +// supplied callback. +func (i *InvoiceStore) UpdateInvoice(ctx context.Context, ref invpkg.InvoiceRef, + _ *invpkg.SetID, callback invpkg.InvoiceUpdateCallback) ( + *invpkg.Invoice, error) { + + var updatedInvoice *invpkg.Invoice + + txOpt := InvoiceQueriesTxOptions{readOnly: false} + txErr := i.db.ExecTx(ctx, &txOpt, func(db InvoiceQueries) error { + invoice, err := i.fetchInvoice(ctx, db, ref) + if err != nil { + return err + } + + updateTime := i.clock.Now() + updater := &sqlInvoiceUpdater{ + db: db, + ctx: ctx, + invoice: invoice, + updateTime: updateTime, + } + + payHash := ref.PayHash() + updatedInvoice, err = invpkg.UpdateInvoice( + payHash, invoice, updateTime, callback, updater, + ) + + return err + }) + if txErr != nil { + // If the invoice is already settled, we'll return the + // (unchanged) invoice and the ErrInvoiceAlreadySettled error. + if errors.Is(txErr, invpkg.ErrInvoiceAlreadySettled) { + return updatedInvoice, txErr + } + + return nil, txErr + } + + return updatedInvoice, nil +} + +// DeleteInvoice attempts to delete the passed invoices and all their related +// data from the database in one transaction. +func (i *InvoiceStore) DeleteInvoice(ctx context.Context, + invoicesToDelete []invpkg.InvoiceDeleteRef) error { + + // All the InvoiceDeleteRef instances include the add index of the + // invoice. The rest was added to ensure that the invoices were deleted + // properly in the kv database. When we have fully migrated we can + // remove the rest of the fields. + for _, ref := range invoicesToDelete { + if ref.AddIndex == 0 { + return fmt.Errorf("unable to delete invoice using a "+ + "ref without AddIndex set: %v", ref) + } + } + + var writeTxOpt InvoiceQueriesTxOptions + err := i.db.ExecTx(ctx, &writeTxOpt, func(db InvoiceQueries) error { + for _, ref := range invoicesToDelete { + params := sqlc.DeleteInvoiceParams{ + AddIndex: sqlInt64(ref.AddIndex), + } + + if ref.SettleIndex != 0 { + params.SettleIndex = sqlInt64(ref.SettleIndex) + } + + if ref.PayHash != lntypes.ZeroHash { + params.Hash = ref.PayHash[:] + } + + result, err := db.DeleteInvoice(ctx, params) + if err != nil { + return fmt.Errorf("unable to delete "+ + "invoice(%v): %w", ref.AddIndex, err) + } + rowsAffected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("unable to get rows "+ + "affected: %w", err) + } + if rowsAffected == 0 { + return fmt.Errorf("%w: %v", + invpkg.ErrInvoiceNotFound, ref.AddIndex) + } + } + + return nil + }) + + if err != nil { + return fmt.Errorf("unable to delete invoices: %w", err) + } + + return nil +} + +// DeleteCanceledInvoices removes all canceled invoices from the database. +func (i *InvoiceStore) DeleteCanceledInvoices(ctx context.Context) error { + var writeTxOpt InvoiceQueriesTxOptions + err := i.db.ExecTx(ctx, &writeTxOpt, func(db InvoiceQueries) error { + _, err := db.DeleteCanceledInvoices(ctx) + if err != nil { + return fmt.Errorf("unable to delete canceled "+ + "invoices: %w", err) + } + + return nil + }) + if err != nil { + return fmt.Errorf("unable to delete invoices: %w", err) + } + + return nil +} + +// fetchInvoiceData fetches additional data for the given invoice. If the +// invoice is AMP and the setID is not nil, then it will also fetch the AMP +// state and HTLCs for the given setID, otherwise for all AMP sub invoices of +// the invoice. If fetchAmpHtlcs is true, it will also fetch the AMP HTLCs. +func fetchInvoiceData(ctx context.Context, db InvoiceQueries, + row sqlc.Invoice, setID *[32]byte, fetchAmpHtlcs bool) (*lntypes.Hash, + *invpkg.Invoice, error) { + + // Unmarshal the common data. + hash, invoice, err := unmarshalInvoice(row) + if err != nil { + return nil, nil, fmt.Errorf("unable to unmarshal "+ + "invoice(id=%d) from db: %w", row.ID, err) + } + + // Fetch the invoice features. + features, err := getInvoiceFeatures(ctx, db, row.ID) + if err != nil { + return nil, nil, err + } + + invoice.Terms.Features = features + + // If this is an AMP invoice, we'll need fetch the AMP state along + // with the HTLCs (if requested). + if invoice.IsAMP() { + invoiceID := int64(invoice.AddIndex) + ampState, ampHtlcs, err := fetchAmpState( + ctx, db, invoiceID, setID, fetchAmpHtlcs, + ) + if err != nil { + return nil, nil, err + } + + invoice.AMPState = ampState + invoice.Htlcs = ampHtlcs + + return hash, invoice, nil + } + + // Otherwise simply fetch the invoice HTLCs. + htlcs, err := getInvoiceHtlcs(ctx, db, row.ID) + if err != nil { + return nil, nil, err + } + + if len(htlcs) > 0 { + invoice.Htlcs = htlcs + var amountPaid lnwire.MilliSatoshi + for _, htlc := range htlcs { + if htlc.State == invpkg.HtlcStateSettled { + amountPaid += htlc.Amt + } + } + invoice.AmtPaid = amountPaid + } + + return hash, invoice, nil +} + +// getInvoiceFeatures fetches the invoice features for the given invoice id. +func getInvoiceFeatures(ctx context.Context, db InvoiceQueries, + invoiceID int64) (*lnwire.FeatureVector, error) { + + rows, err := db.GetInvoiceFeatures(ctx, invoiceID) + if err != nil { + return nil, fmt.Errorf("unable to get invoice features: %w", + err) + } + + features := lnwire.EmptyFeatureVector() + for _, feature := range rows { + features.Set(lnwire.FeatureBit(feature.Feature)) + } + + return features, nil +} + +// getInvoiceHtlcs fetches the invoice htlcs for the given invoice id. +func getInvoiceHtlcs(ctx context.Context, db InvoiceQueries, + invoiceID int64) (map[invpkg.CircuitKey]*invpkg.InvoiceHTLC, error) { + + htlcRows, err := db.GetInvoiceHTLCs(ctx, invoiceID) + if err != nil { + return nil, fmt.Errorf("unable to get invoice htlcs: %w", err) + } + + // We have no htlcs to unmarshal. + if len(htlcRows) == 0 { + return nil, nil + } + + crRows, err := db.GetInvoiceHTLCCustomRecords(ctx, invoiceID) + if err != nil { + return nil, fmt.Errorf("unable to get custom records for "+ + "invoice htlcs: %w", err) + } + + cr := make(map[int64]record.CustomSet, len(crRows)) + for _, row := range crRows { + if _, ok := cr[row.HtlcID]; !ok { + cr[row.HtlcID] = make(record.CustomSet) + } + + value := row.Value + if value == nil { + value = []byte{} + } + cr[row.HtlcID][uint64(row.Key)] = value + } + + htlcs := make(map[invpkg.CircuitKey]*invpkg.InvoiceHTLC, len(htlcRows)) + + for _, row := range htlcRows { + circuiteKey, htlc, err := unmarshalInvoiceHTLC(row) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal "+ + "htlc(%d): %w", row.ID, err) + } + + if customRecords, ok := cr[row.ID]; ok { + htlc.CustomRecords = customRecords + } else { + htlc.CustomRecords = make(record.CustomSet) + } + + htlcs[circuiteKey] = htlc + } + + return htlcs, nil +} + +// unmarshalInvoice converts an InvoiceRow to an Invoice. +func unmarshalInvoice(row sqlc.Invoice) (*lntypes.Hash, *invpkg.Invoice, + error) { + + var ( + settleIndex int64 + settledAt time.Time + memo []byte + paymentRequest []byte + preimage *lntypes.Preimage + paymentAddr [32]byte + ) + + hash, err := lntypes.MakeHash(row.Hash) + if err != nil { + return nil, nil, err + } + + if row.SettleIndex.Valid { + settleIndex = row.SettleIndex.Int64 + } + + if row.SettledAt.Valid { + settledAt = row.SettledAt.Time.Local() + } + + if row.Memo.Valid { + memo = []byte(row.Memo.String) + } + + // Keysend payments will have this field empty. + if row.PaymentRequest.Valid { + paymentRequest = []byte(row.PaymentRequest.String) + } else { + paymentRequest = []byte{} + } + + // We may not have the preimage if this a hodl invoice. + if row.Preimage != nil { + preimage = &lntypes.Preimage{} + copy(preimage[:], row.Preimage) + } + + copy(paymentAddr[:], row.PaymentAddr) + + var cltvDelta int32 + if row.CltvDelta.Valid { + cltvDelta = row.CltvDelta.Int32 + } + + invoice := &invpkg.Invoice{ + SettleIndex: uint64(settleIndex), + SettleDate: settledAt, + Memo: memo, + PaymentRequest: paymentRequest, + CreationDate: row.CreatedAt.Local(), + Terms: invpkg.ContractTerm{ + FinalCltvDelta: cltvDelta, + Expiry: time.Duration(row.Expiry), + PaymentPreimage: preimage, + Value: lnwire.MilliSatoshi(row.AmountMsat), + PaymentAddr: paymentAddr, + }, + AddIndex: uint64(row.ID), + State: invpkg.ContractState(row.State), + AmtPaid: lnwire.MilliSatoshi(row.AmountPaidMsat), + Htlcs: make(map[models.CircuitKey]*invpkg.InvoiceHTLC), + AMPState: invpkg.AMPInvoiceState{}, + HodlInvoice: row.IsHodl, + } + + return &hash, invoice, nil +} + +// unmarshalInvoiceHTLC converts an sqlc.InvoiceHtlc to an InvoiceHTLC. +func unmarshalInvoiceHTLC(row sqlc.InvoiceHtlc) (invpkg.CircuitKey, + *invpkg.InvoiceHTLC, error) { + + uint64ChanID, err := strconv.ParseUint(row.ChanID, 10, 64) + if err != nil { + return invpkg.CircuitKey{}, nil, err + } + + chanID := lnwire.NewShortChanIDFromInt(uint64ChanID) + + if row.HtlcID < 0 { + return invpkg.CircuitKey{}, nil, fmt.Errorf("invalid uint64 "+ + "value: %v", row.HtlcID) + } + + htlcID := uint64(row.HtlcID) + + circuitKey := invpkg.CircuitKey{ + ChanID: chanID, + HtlcID: htlcID, + } + + htlc := &invpkg.InvoiceHTLC{ + Amt: lnwire.MilliSatoshi(row.AmountMsat), + AcceptHeight: uint32(row.AcceptHeight), + AcceptTime: row.AcceptTime.Local(), + Expiry: uint32(row.ExpiryHeight), + State: invpkg.HtlcState(row.State), + } + + if row.TotalMppMsat.Valid { + htlc.MppTotalAmt = lnwire.MilliSatoshi(row.TotalMppMsat.Int64) + } + + if row.ResolveTime.Valid { + htlc.ResolveTime = row.ResolveTime.Time.Local() + } + + return circuitKey, htlc, nil +} + +// queryWithLimit is a helper method that can be used to query the database +// using a limit and offset. The passed query function should return the number +// of rows returned and an error if any. +func queryWithLimit(query func(int) (int, error), limit int) error { + offset := 0 + for { + rows, err := query(offset) + if err != nil { + return err + } + + if rows < limit { + return nil + } + + offset += limit + } +} diff --git a/sqldb/no_sqlite.go b/sqldb/no_sqlite.go new file mode 100644 index 000000000..9ea35c43c --- /dev/null +++ b/sqldb/no_sqlite.go @@ -0,0 +1,18 @@ +//go:build js || (windows && (arm || 386)) || (linux && (ppc64 || mips || mipsle || mips64)) + +package sqldb + +import "fmt" + +// SqliteStore is a database store implementation that uses a sqlite backend. +type SqliteStore struct { + cfg *SqliteConfig + + *BaseDB +} + +// NewSqliteStore attempts to open a new sqlite database based on the passed +// config. +func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { + return nil, fmt.Errorf("SQLite backend not supported in WebAssembly") +} diff --git a/sqldb/postgres.go b/sqldb/postgres.go index 3cfa59efc..e6e88c93b 100644 --- a/sqldb/postgres.go +++ b/sqldb/postgres.go @@ -2,18 +2,14 @@ package sqldb import ( "database/sql" - "fmt" - "testing" + "net/url" + "path" + "strings" "time" postgres_migrate "github.com/golang-migrate/migrate/v4/database/postgres" _ "github.com/golang-migrate/migrate/v4/source/file" // Read migrations from files. // nolint:lll "github.com/lightningnetwork/lnd/sqldb/sqlc" - "github.com/stretchr/testify/require" -) - -const ( - dsnTemplate = "postgres://%v:%v@%v:%d/%v?sslmode=%v" ) var ( @@ -25,35 +21,45 @@ var ( DefaultPostgresFixtureLifetime = 10 * time.Minute ) -// PostgresConfig holds the postgres database configuration. -// -//nolint:lll -type PostgresConfig struct { - SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` - Host string `long:"host" description:"Database server hostname."` - Port int `long:"port" description:"Database server port."` - User string `long:"user" description:"Database user."` - Password string `long:"password" description:"Database user's password."` - DBName string `long:"dbname" description:"Database name to use."` - MaxOpenConnections int `long:"maxconnections" description:"Max open connections to keep alive to the database server."` - RequireSSL bool `long:"requiressl" description:"Whether to require using SSL (mode: require) when connecting to the server."` +// replacePasswordInDSN takes a DSN string and returns it with the password +// replaced by "***". +func replacePasswordInDSN(dsn string) (string, error) { + // Parse the DSN as a URL + u, err := url.Parse(dsn) + if err != nil { + return "", err + } + + // Check if the URL has a user info part + if u.User != nil { + username := u.User.Username() + + // Reconstruct user info with "***" as password + userInfo := username + ":***@" + + // Rebuild the DSN with the modified user info + sanitizeDSN := strings.Replace( + dsn, u.User.String()+"@", userInfo, 1, + ) + + return sanitizeDSN, nil + } + + // Return the original DSN if no user info is present + return dsn, nil } -// DSN returns the dns to connect to the database. -func (s *PostgresConfig) DSN(hidePassword bool) string { - var sslMode = "disable" - if s.RequireSSL { - sslMode = "require" +// getDatabaseNameFromDSN extracts the database name from a DSN string. +func getDatabaseNameFromDSN(dsn string) (string, error) { + // Parse the DSN as a URL + u, err := url.Parse(dsn) + if err != nil { + return "", err } - password := s.Password - if hidePassword { - // Placeholder used for logging the DSN safely. - password = "****" - } - - return fmt.Sprintf(dsnTemplate, s.User, password, s.Host, s.Port, - s.DBName, sslMode) + // The database name is the last segment of the path. Trim leading slash + // and return the last segment. + return path.Base(u.Path), nil } // PostgresStore is a database store implementation that uses a Postgres @@ -67,16 +73,25 @@ type PostgresStore struct { // NewPostgresStore creates a new store that is backed by a Postgres database // backend. func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { - log.Infof("Using SQL database '%s'", cfg.DSN(true)) + sanitizedDSN, err := replacePasswordInDSN(cfg.Dsn) + if err != nil { + return nil, err + } + log.Infof("Using SQL database '%s'", sanitizedDSN) - rawDB, err := sql.Open("pgx", cfg.DSN(false)) + dbName, err := getDatabaseNameFromDSN(cfg.Dsn) + if err != nil { + return nil, err + } + + rawDB, err := sql.Open("pgx", cfg.Dsn) if err != nil { return nil, err } maxConns := defaultMaxConns - if cfg.MaxOpenConnections > 0 { - maxConns = cfg.MaxOpenConnections + if cfg.MaxConnections > 0 { + maxConns = cfg.MaxConnections } rawDB.SetMaxOpenConns(maxConns) @@ -89,7 +104,7 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { // system. // // First, we'll need to open up a new migration instance for - // our current target database: sqlite. + // our current target database: Postgres. driver, err := postgres_migrate.WithInstance( rawDB, &postgres_migrate.Config{}, ) @@ -100,11 +115,12 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { postgresFS := newReplacerFS(sqlSchemas, map[string]string{ "BLOB": "BYTEA", "INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY", + "BIGINT PRIMARY KEY": "BIGSERIAL PRIMARY KEY", "TIMESTAMP": "TIMESTAMP WITHOUT TIME ZONE", }) err = applyMigrations( - postgresFS, driver, "sqlc/migrations", cfg.DBName, + postgresFS, driver, "sqlc/migrations", dbName, ) if err != nil { return nil, err @@ -121,21 +137,3 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { }, }, nil } - -// NewTestPostgresDB is a helper function that creates a Postgres database for -// testing. -func NewTestPostgresDB(t *testing.T) *PostgresStore { - t.Helper() - - t.Logf("Creating new Postgres DB for testing") - - sqlFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime) - store, err := NewPostgresStore(sqlFixture.GetConfig()) - require.NoError(t, err) - - t.Cleanup(func() { - sqlFixture.TearDown(t) - }) - - return store -} diff --git a/sqldb/postgres_fixture.go b/sqldb/postgres_fixture.go index 991aad980..a0f0893ce 100644 --- a/sqldb/postgres_fixture.go +++ b/sqldb/postgres_fixture.go @@ -1,8 +1,12 @@ +//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) && !(netbsd || openbsd) + package sqldb import ( "context" + "crypto/rand" "database/sql" + "encoding/hex" "fmt" "strconv" "strings" @@ -55,6 +59,7 @@ func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture { "postgres", "-c", "log_statement=all", "-c", "log_destination=stderr", + "-c", "max_connections=1000", }, }, func(config *docker.HostConfig) { // Set AutoRemove to true so that stopped container goes away @@ -74,7 +79,7 @@ func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture { host: host, port: int(port), } - databaseURL := fixture.GetDSN() + databaseURL := fixture.GetConfig(testPgDBName).Dsn log.Infof("Connecting to Postgres fixture: %v\n", databaseURL) // Tell docker to hard kill the container in "expiry" seconds. @@ -103,20 +108,13 @@ func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture { return fixture } -// GetDSN returns the DSN (Data Source Name) for the started Postgres node. -func (f *TestPgFixture) GetDSN() string { - return f.GetConfig().DSN(false) -} - // GetConfig returns the full config of the Postgres node. -func (f *TestPgFixture) GetConfig() *PostgresConfig { +func (f *TestPgFixture) GetConfig(dbName string) *PostgresConfig { return &PostgresConfig{ - Host: f.host, - Port: f.port, - User: testPgUser, - Password: testPgPass, - DBName: testPgDBName, - RequireSSL: false, + Dsn: fmt.Sprintf( + "postgres://%v:%v@%v:%v/%v?sslmode=disable", + testPgUser, testPgPass, f.host, f.port, dbName, + ), } } @@ -126,15 +124,32 @@ func (f *TestPgFixture) TearDown(t *testing.T) { require.NoError(t, err, "Could not purge resource") } -// ClearDB clears the database. -func (f *TestPgFixture) ClearDB(t *testing.T) { - dbConn, err := sql.Open("postgres", f.GetDSN()) +// NewTestPostgresDB is a helper function that creates a Postgres database for +// testing using the given fixture. +func NewTestPostgresDB(t *testing.T, fixture *TestPgFixture) *PostgresStore { + t.Helper() + + // Create random database name. + randBytes := make([]byte, 8) + _, err := rand.Read(randBytes) + if err != nil { + t.Fatal(err) + } + + dbName := "test_" + hex.EncodeToString(randBytes) + + t.Logf("Creating new Postgres DB '%s' for testing", dbName) + + _, err = fixture.db.ExecContext( + context.Background(), "CREATE DATABASE "+dbName, + ) + if err != nil { + t.Fatal(err) + } + + cfg := fixture.GetConfig(dbName) + store, err := NewPostgresStore(cfg) require.NoError(t, err) - _, err = dbConn.ExecContext( - context.Background(), - `DROP SCHEMA IF EXISTS public CASCADE; - CREATE SCHEMA public;`, - ) - require.NoError(t, err) + return store } diff --git a/sqldb/sqlc/amp_invoices.sql.go b/sqldb/sqlc/amp_invoices.sql.go index c803a8307..3fcfe4b27 100644 --- a/sqldb/sqlc/amp_invoices.sql.go +++ b/sqldb/sqlc/amp_invoices.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 // source: amp_invoices.sql package sqlc @@ -11,67 +11,69 @@ import ( "time" ) -const deleteAMPHTLCCustomRecords = `-- name: DeleteAMPHTLCCustomRecords :exec -WITH htlc_ids AS ( - SELECT htlc_id - FROM amp_invoice_htlcs - WHERE invoice_id = $1 +const fetchAMPSubInvoiceHTLCs = `-- name: FetchAMPSubInvoiceHTLCs :many +SELECT + amp.set_id, amp.root_share, amp.child_index, amp.hash, amp.preimage, + invoice_htlcs.id, invoice_htlcs.chan_id, invoice_htlcs.htlc_id, invoice_htlcs.amount_msat, invoice_htlcs.total_mpp_msat, invoice_htlcs.accept_height, invoice_htlcs.accept_time, invoice_htlcs.expiry_height, invoice_htlcs.state, invoice_htlcs.resolve_time, invoice_htlcs.invoice_id +FROM amp_sub_invoice_htlcs amp +INNER JOIN invoice_htlcs ON amp.htlc_id = invoice_htlcs.id +WHERE amp.invoice_id = $1 +AND ( + set_id = $2 OR + $2 IS NULL ) -DELETE -FROM invoice_htlc_custom_records -WHERE htlc_id IN (SELECT id FROM htlc_ids) ` -func (q *Queries) DeleteAMPHTLCCustomRecords(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteAMPHTLCCustomRecords, invoiceID) - return err +type FetchAMPSubInvoiceHTLCsParams struct { + InvoiceID int64 + SetID []byte } -const deleteAMPHTLCs = `-- name: DeleteAMPHTLCs :exec -DELETE -FROM amp_invoice_htlcs -WHERE invoice_id = $1 -` - -func (q *Queries) DeleteAMPHTLCs(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteAMPHTLCs, invoiceID) - return err +type FetchAMPSubInvoiceHTLCsRow struct { + SetID []byte + RootShare []byte + ChildIndex int64 + Hash []byte + Preimage []byte + ID int64 + ChanID string + HtlcID int64 + AmountMsat int64 + TotalMppMsat sql.NullInt64 + AcceptHeight int32 + AcceptTime time.Time + ExpiryHeight int32 + State int16 + ResolveTime sql.NullTime + InvoiceID int64 } -const deleteAMPInvoiceHTLC = `-- name: DeleteAMPInvoiceHTLC :exec -DELETE -FROM amp_invoice_htlcs -WHERE set_id = $1 -` - -func (q *Queries) DeleteAMPInvoiceHTLC(ctx context.Context, setID []byte) error { - _, err := q.db.ExecContext(ctx, deleteAMPInvoiceHTLC, setID) - return err -} - -const getAMPInvoiceHTLCsByInvoiceID = `-- name: GetAMPInvoiceHTLCsByInvoiceID :many -SELECT set_id, htlc_id, invoice_id, root_share, child_index, hash, preimage -FROM amp_invoice_htlcs -WHERE invoice_id = $1 -` - -func (q *Queries) GetAMPInvoiceHTLCsByInvoiceID(ctx context.Context, invoiceID int32) ([]AmpInvoiceHtlc, error) { - rows, err := q.db.QueryContext(ctx, getAMPInvoiceHTLCsByInvoiceID, invoiceID) +func (q *Queries) FetchAMPSubInvoiceHTLCs(ctx context.Context, arg FetchAMPSubInvoiceHTLCsParams) ([]FetchAMPSubInvoiceHTLCsRow, error) { + rows, err := q.db.QueryContext(ctx, fetchAMPSubInvoiceHTLCs, arg.InvoiceID, arg.SetID) if err != nil { return nil, err } defer rows.Close() - var items []AmpInvoiceHtlc + var items []FetchAMPSubInvoiceHTLCsRow for rows.Next() { - var i AmpInvoiceHtlc + var i FetchAMPSubInvoiceHTLCsRow if err := rows.Scan( &i.SetID, - &i.HtlcID, - &i.InvoiceID, &i.RootShare, &i.ChildIndex, &i.Hash, &i.Preimage, + &i.ID, + &i.ChanID, + &i.HtlcID, + &i.AmountMsat, + &i.TotalMppMsat, + &i.AcceptHeight, + &i.AcceptTime, + &i.ExpiryHeight, + &i.State, + &i.ResolveTime, + &i.InvoiceID, ); err != nil { return nil, err } @@ -86,29 +88,37 @@ func (q *Queries) GetAMPInvoiceHTLCsByInvoiceID(ctx context.Context, invoiceID i return items, nil } -const getAMPInvoiceHTLCsBySetID = `-- name: GetAMPInvoiceHTLCsBySetID :many -SELECT set_id, htlc_id, invoice_id, root_share, child_index, hash, preimage -FROM amp_invoice_htlcs -WHERE set_id = $1 +const fetchAMPSubInvoices = `-- name: FetchAMPSubInvoices :many +SELECT set_id, state, created_at, settled_at, settle_index, invoice_id +FROM amp_sub_invoices +WHERE invoice_id = $1 +AND ( + set_id = $2 OR + $2 IS NULL +) ` -func (q *Queries) GetAMPInvoiceHTLCsBySetID(ctx context.Context, setID []byte) ([]AmpInvoiceHtlc, error) { - rows, err := q.db.QueryContext(ctx, getAMPInvoiceHTLCsBySetID, setID) +type FetchAMPSubInvoicesParams struct { + InvoiceID int64 + SetID []byte +} + +func (q *Queries) FetchAMPSubInvoices(ctx context.Context, arg FetchAMPSubInvoicesParams) ([]AmpSubInvoice, error) { + rows, err := q.db.QueryContext(ctx, fetchAMPSubInvoices, arg.InvoiceID, arg.SetID) if err != nil { return nil, err } defer rows.Close() - var items []AmpInvoiceHtlc + var items []AmpSubInvoice for rows.Next() { - var i AmpInvoiceHtlc + var i AmpSubInvoice if err := rows.Scan( &i.SetID, - &i.HtlcID, + &i.State, + &i.CreatedAt, + &i.SettledAt, + &i.SettleIndex, &i.InvoiceID, - &i.RootShare, - &i.ChildIndex, - &i.Hash, - &i.Preimage, ); err != nil { return nil, err } @@ -123,28 +133,84 @@ func (q *Queries) GetAMPInvoiceHTLCsBySetID(ctx context.Context, setID []byte) ( return items, nil } -const getSetIDHTLCsCustomRecords = `-- name: GetSetIDHTLCsCustomRecords :many -SELECT ihcr.htlc_id, key, value -FROM amp_invoice_htlcs aih JOIN invoice_htlc_custom_records ihcr ON aih.id=ihcr.htlc_id -WHERE aih.set_id = $1 +const fetchSettledAMPSubInvoices = `-- name: FetchSettledAMPSubInvoices :many +SELECT + a.set_id, + a.settle_index as amp_settle_index, + a.settled_at as amp_settled_at, + i.id, i.hash, i.preimage, i.settle_index, i.settled_at, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.payment_request_hash, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at +FROM amp_sub_invoices a +INNER JOIN invoices i ON a.invoice_id = i.id +WHERE ( + a.settle_index >= $1 OR + $1 IS NULL +) AND ( + a.settle_index <= $2 OR + $2 IS NULL +) ` -type GetSetIDHTLCsCustomRecordsRow struct { - HtlcID int64 - Key int64 - Value []byte +type FetchSettledAMPSubInvoicesParams struct { + SettleIndexGet sql.NullInt64 + SettleIndexLet sql.NullInt64 } -func (q *Queries) GetSetIDHTLCsCustomRecords(ctx context.Context, setID []byte) ([]GetSetIDHTLCsCustomRecordsRow, error) { - rows, err := q.db.QueryContext(ctx, getSetIDHTLCsCustomRecords, setID) +type FetchSettledAMPSubInvoicesRow struct { + SetID []byte + AmpSettleIndex sql.NullInt64 + AmpSettledAt sql.NullTime + ID int64 + Hash []byte + Preimage []byte + SettleIndex sql.NullInt64 + SettledAt sql.NullTime + Memo sql.NullString + AmountMsat int64 + CltvDelta sql.NullInt32 + Expiry int32 + PaymentAddr []byte + PaymentRequest sql.NullString + PaymentRequestHash []byte + State int16 + AmountPaidMsat int64 + IsAmp bool + IsHodl bool + IsKeysend bool + CreatedAt time.Time +} + +func (q *Queries) FetchSettledAMPSubInvoices(ctx context.Context, arg FetchSettledAMPSubInvoicesParams) ([]FetchSettledAMPSubInvoicesRow, error) { + rows, err := q.db.QueryContext(ctx, fetchSettledAMPSubInvoices, arg.SettleIndexGet, arg.SettleIndexLet) if err != nil { return nil, err } defer rows.Close() - var items []GetSetIDHTLCsCustomRecordsRow + var items []FetchSettledAMPSubInvoicesRow for rows.Next() { - var i GetSetIDHTLCsCustomRecordsRow - if err := rows.Scan(&i.HtlcID, &i.Key, &i.Value); err != nil { + var i FetchSettledAMPSubInvoicesRow + if err := rows.Scan( + &i.SetID, + &i.AmpSettleIndex, + &i.AmpSettledAt, + &i.ID, + &i.Hash, + &i.Preimage, + &i.SettleIndex, + &i.SettledAt, + &i.Memo, + &i.AmountMsat, + &i.CltvDelta, + &i.Expiry, + &i.PaymentAddr, + &i.PaymentRequest, + &i.PaymentRequestHash, + &i.State, + &i.AmountPaidMsat, + &i.IsAmp, + &i.IsHodl, + &i.IsKeysend, + &i.CreatedAt, + ); err != nil { return nil, err } items = append(items, i) @@ -158,15 +224,27 @@ func (q *Queries) GetSetIDHTLCsCustomRecords(ctx context.Context, setID []byte) return items, nil } -const insertAMPInvoiceHTLC = `-- name: InsertAMPInvoiceHTLC :exec -INSERT INTO amp_invoice_htlcs ( - set_id, htlc_id, root_share, child_index, hash, preimage +const getAMPInvoiceID = `-- name: GetAMPInvoiceID :one +SELECT invoice_id FROM amp_sub_invoices WHERE set_id = $1 +` + +func (q *Queries) GetAMPInvoiceID(ctx context.Context, setID []byte) (int64, error) { + row := q.db.QueryRowContext(ctx, getAMPInvoiceID, setID) + var invoice_id int64 + err := row.Scan(&invoice_id) + return invoice_id, err +} + +const insertAMPSubInvoiceHTLC = `-- name: InsertAMPSubInvoiceHTLC :exec +INSERT INTO amp_sub_invoice_htlcs ( + invoice_id, set_id, htlc_id, root_share, child_index, hash, preimage ) VALUES ( - $1, $2, $3, $4, $5, $6 + $1, $2, $3, $4, $5, $6, $7 ) ` -type InsertAMPInvoiceHTLCParams struct { +type InsertAMPSubInvoiceHTLCParams struct { + InvoiceID int64 SetID []byte HtlcID int64 RootShare []byte @@ -175,8 +253,9 @@ type InsertAMPInvoiceHTLCParams struct { Preimage []byte } -func (q *Queries) InsertAMPInvoiceHTLC(ctx context.Context, arg InsertAMPInvoiceHTLCParams) error { - _, err := q.db.ExecContext(ctx, insertAMPInvoiceHTLC, +func (q *Queries) InsertAMPSubInvoiceHTLC(ctx context.Context, arg InsertAMPSubInvoiceHTLCParams) error { + _, err := q.db.ExecContext(ctx, insertAMPSubInvoiceHTLC, + arg.InvoiceID, arg.SetID, arg.HtlcID, arg.RootShare, @@ -187,140 +266,75 @@ func (q *Queries) InsertAMPInvoiceHTLC(ctx context.Context, arg InsertAMPInvoice return err } -const insertAMPInvoicePayment = `-- name: InsertAMPInvoicePayment :exec -INSERT INTO amp_invoice_payments ( - set_id, state, created_at, settled_index, invoice_id -) VALUES ( - $1, $2, $3, $4, $5 +const updateAMPSubInvoiceHTLCPreimage = `-- name: UpdateAMPSubInvoiceHTLCPreimage :execresult +UPDATE amp_sub_invoice_htlcs AS a +SET preimage = $4 +WHERE a.invoice_id = $1 AND a.set_id = $2 AND a.htlc_id = ( + SELECT id FROM invoice_htlcs AS i WHERE i.htlc_id = $3 ) ` -type InsertAMPInvoicePaymentParams struct { - SetID []byte - State int16 - CreatedAt time.Time - SettledIndex sql.NullInt32 - InvoiceID int32 +type UpdateAMPSubInvoiceHTLCPreimageParams struct { + InvoiceID int64 + SetID []byte + HtlcID int64 + Preimage []byte } -func (q *Queries) InsertAMPInvoicePayment(ctx context.Context, arg InsertAMPInvoicePaymentParams) error { - _, err := q.db.ExecContext(ctx, insertAMPInvoicePayment, +func (q *Queries) UpdateAMPSubInvoiceHTLCPreimage(ctx context.Context, arg UpdateAMPSubInvoiceHTLCPreimageParams) (sql.Result, error) { + return q.db.ExecContext(ctx, updateAMPSubInvoiceHTLCPreimage, + arg.InvoiceID, + arg.SetID, + arg.HtlcID, + arg.Preimage, + ) +} + +const updateAMPSubInvoiceState = `-- name: UpdateAMPSubInvoiceState :exec +UPDATE amp_sub_invoices +SET state = $2, + settle_index = COALESCE(settle_index, $3), + settled_at = COALESCE(settled_at, $4) +WHERE set_id = $1 +` + +type UpdateAMPSubInvoiceStateParams struct { + SetID []byte + State int16 + SettleIndex sql.NullInt64 + SettledAt sql.NullTime +} + +func (q *Queries) UpdateAMPSubInvoiceState(ctx context.Context, arg UpdateAMPSubInvoiceStateParams) error { + _, err := q.db.ExecContext(ctx, updateAMPSubInvoiceState, + arg.SetID, + arg.State, + arg.SettleIndex, + arg.SettledAt, + ) + return err +} + +const upsertAMPSubInvoice = `-- name: UpsertAMPSubInvoice :execresult +INSERT INTO amp_sub_invoices ( + set_id, state, created_at, invoice_id +) VALUES ( + $1, $2, $3, $4 +) ON CONFLICT (set_id, invoice_id) DO NOTHING +` + +type UpsertAMPSubInvoiceParams struct { + SetID []byte + State int16 + CreatedAt time.Time + InvoiceID int64 +} + +func (q *Queries) UpsertAMPSubInvoice(ctx context.Context, arg UpsertAMPSubInvoiceParams) (sql.Result, error) { + return q.db.ExecContext(ctx, upsertAMPSubInvoice, arg.SetID, arg.State, arg.CreatedAt, - arg.SettledIndex, arg.InvoiceID, ) - return err -} - -const selectAMPInvoicePayments = `-- name: SelectAMPInvoicePayments :many -SELECT aip.set_id, aip.state, aip.created_at, aip.settled_index, aip.invoice_id, ip.id, ip.settled_at, ip.amount_paid_msat, ip.invoice_id -FROM amp_invoice_payments aip LEFT JOIN invoice_payments ip ON aip.settled_index = ip.id -WHERE ( - set_id = $1 OR - $1 IS NULL -) AND ( - aip.settled_index = $2 OR - $2 IS NULL -) AND ( - aip.invoice_id = $3 OR - $3 IS NULL -) -` - -type SelectAMPInvoicePaymentsParams struct { - SetID []byte - SettledIndex sql.NullInt32 - InvoiceID sql.NullInt32 -} - -type SelectAMPInvoicePaymentsRow struct { - SetID []byte - State int16 - CreatedAt time.Time - SettledIndex sql.NullInt32 - InvoiceID int32 - ID sql.NullInt32 - SettledAt sql.NullTime - AmountPaidMsat sql.NullInt64 - InvoiceID_2 sql.NullInt32 -} - -func (q *Queries) SelectAMPInvoicePayments(ctx context.Context, arg SelectAMPInvoicePaymentsParams) ([]SelectAMPInvoicePaymentsRow, error) { - rows, err := q.db.QueryContext(ctx, selectAMPInvoicePayments, arg.SetID, arg.SettledIndex, arg.InvoiceID) - if err != nil { - return nil, err - } - defer rows.Close() - var items []SelectAMPInvoicePaymentsRow - for rows.Next() { - var i SelectAMPInvoicePaymentsRow - if err := rows.Scan( - &i.SetID, - &i.State, - &i.CreatedAt, - &i.SettledIndex, - &i.InvoiceID, - &i.ID, - &i.SettledAt, - &i.AmountPaidMsat, - &i.InvoiceID_2, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - -const updateAMPInvoiceHTLC = `-- name: UpdateAMPInvoiceHTLC :exec -UPDATE amp_invoice_htlcs -SET preimage = $1 -WHERE htlc_id = $2 -` - -type UpdateAMPInvoiceHTLCParams struct { - Preimage []byte - HtlcID int64 -} - -func (q *Queries) UpdateAMPInvoiceHTLC(ctx context.Context, arg UpdateAMPInvoiceHTLCParams) error { - _, err := q.db.ExecContext(ctx, updateAMPInvoiceHTLC, arg.Preimage, arg.HtlcID) - return err -} - -const updateAMPPayment = `-- name: UpdateAMPPayment :exec -UPDATE amp_invoice_payments -SET state = $1, settled_index = $2 -WHERE state = 0 AND ( - set_id = $3 OR - $3 IS NULL -) AND ( - invoice_id = $4 OR - $4 IS NULL -) -` - -type UpdateAMPPaymentParams struct { - State int16 - SettledIndex sql.NullInt32 - SetID []byte - InvoiceID sql.NullInt32 -} - -func (q *Queries) UpdateAMPPayment(ctx context.Context, arg UpdateAMPPaymentParams) error { - _, err := q.db.ExecContext(ctx, updateAMPPayment, - arg.State, - arg.SettledIndex, - arg.SetID, - arg.InvoiceID, - ) - return err } diff --git a/sqldb/sqlc/db.go b/sqldb/sqlc/db.go index 9848f3e6a..8ed64d139 100644 --- a/sqldb/sqlc/db.go +++ b/sqldb/sqlc/db.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 package sqlc diff --git a/sqldb/sqlc/invoice_events.sql.go b/sqldb/sqlc/invoice_events.sql.go index 054d9091c..983dfbb15 100644 --- a/sqldb/sqlc/invoice_events.sql.go +++ b/sqldb/sqlc/invoice_events.sql.go @@ -1,128 +1,122 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 // source: invoice_events.sql package sqlc import ( "context" - "database/sql" "time" ) -const deleteInvoiceEvents = `-- name: DeleteInvoiceEvents :exec -DELETE -FROM invoice_events -WHERE invoice_id = $1 -` - -func (q *Queries) DeleteInvoiceEvents(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteInvoiceEvents, invoiceID) - return err -} - -const insertInvoiceEvent = `-- name: InsertInvoiceEvent :exec +const onAMPSubInvoiceCanceled = `-- name: OnAMPSubInvoiceCanceled :exec INSERT INTO invoice_events ( - created_at, invoice_id, htlc_id, set_id, event_type, event_metadata + added_at, event_type, invoice_id, set_id ) VALUES ( - $1, $2, $3, $4, $5, $6 + $1, 4, $2, $3 ) ` -type InsertInvoiceEventParams struct { - CreatedAt time.Time - InvoiceID int32 - HtlcID sql.NullInt64 - SetID []byte - EventType int32 - EventMetadata []byte +type OnAMPSubInvoiceCanceledParams struct { + AddedAt time.Time + InvoiceID int64 + SetID []byte } -func (q *Queries) InsertInvoiceEvent(ctx context.Context, arg InsertInvoiceEventParams) error { - _, err := q.db.ExecContext(ctx, insertInvoiceEvent, - arg.CreatedAt, - arg.InvoiceID, - arg.HtlcID, - arg.SetID, - arg.EventType, - arg.EventMetadata, - ) +func (q *Queries) OnAMPSubInvoiceCanceled(ctx context.Context, arg OnAMPSubInvoiceCanceledParams) error { + _, err := q.db.ExecContext(ctx, onAMPSubInvoiceCanceled, arg.AddedAt, arg.InvoiceID, arg.SetID) return err } -const selectInvoiceEvents = `-- name: SelectInvoiceEvents :many -SELECT id, created_at, invoice_id, htlc_id, set_id, event_type, event_metadata -FROM invoice_events -WHERE ( - invoice_id = $1 OR - $1 IS NULL -) AND ( - htlc_id = $2 OR - $2 IS NULL -) AND ( - set_id = $3 OR - $3 IS NULL -) AND ( - event_type = $4 OR - $4 IS NULL -) AND ( - created_at >= $5 OR - $5 IS NULL -) AND ( - created_at <= $6 OR - $6 IS NULL -) -LIMIT $8 OFFSET $7 +const onAMPSubInvoiceCreated = `-- name: OnAMPSubInvoiceCreated :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id, set_id +) VALUES ( + $1, 3, $2, $3 +) ` -type SelectInvoiceEventsParams struct { - InvoiceID sql.NullInt32 - HtlcID sql.NullInt64 - SetID []byte - EventType sql.NullInt32 - CreatedAfter sql.NullTime - CreatedBefore sql.NullTime - NumOffset int32 - NumLimit int32 +type OnAMPSubInvoiceCreatedParams struct { + AddedAt time.Time + InvoiceID int64 + SetID []byte } -func (q *Queries) SelectInvoiceEvents(ctx context.Context, arg SelectInvoiceEventsParams) ([]InvoiceEvent, error) { - rows, err := q.db.QueryContext(ctx, selectInvoiceEvents, - arg.InvoiceID, - arg.HtlcID, - arg.SetID, - arg.EventType, - arg.CreatedAfter, - arg.CreatedBefore, - arg.NumOffset, - arg.NumLimit, - ) - if err != nil { - return nil, err - } - defer rows.Close() - var items []InvoiceEvent - for rows.Next() { - var i InvoiceEvent - if err := rows.Scan( - &i.ID, - &i.CreatedAt, - &i.InvoiceID, - &i.HtlcID, - &i.SetID, - &i.EventType, - &i.EventMetadata, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil +func (q *Queries) OnAMPSubInvoiceCreated(ctx context.Context, arg OnAMPSubInvoiceCreatedParams) error { + _, err := q.db.ExecContext(ctx, onAMPSubInvoiceCreated, arg.AddedAt, arg.InvoiceID, arg.SetID) + return err +} + +const onAMPSubInvoiceSettled = `-- name: OnAMPSubInvoiceSettled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id, set_id +) VALUES ( + $1, 5, $2, $3 +) +` + +type OnAMPSubInvoiceSettledParams struct { + AddedAt time.Time + InvoiceID int64 + SetID []byte +} + +func (q *Queries) OnAMPSubInvoiceSettled(ctx context.Context, arg OnAMPSubInvoiceSettledParams) error { + _, err := q.db.ExecContext(ctx, onAMPSubInvoiceSettled, arg.AddedAt, arg.InvoiceID, arg.SetID) + return err +} + +const onInvoiceCanceled = `-- name: OnInvoiceCanceled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id +) VALUES ( + $1, 1, $2 +) +` + +type OnInvoiceCanceledParams struct { + AddedAt time.Time + InvoiceID int64 +} + +func (q *Queries) OnInvoiceCanceled(ctx context.Context, arg OnInvoiceCanceledParams) error { + _, err := q.db.ExecContext(ctx, onInvoiceCanceled, arg.AddedAt, arg.InvoiceID) + return err +} + +const onInvoiceCreated = `-- name: OnInvoiceCreated :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id +) VALUES ( + $1, 0, $2 +) +` + +type OnInvoiceCreatedParams struct { + AddedAt time.Time + InvoiceID int64 +} + +func (q *Queries) OnInvoiceCreated(ctx context.Context, arg OnInvoiceCreatedParams) error { + _, err := q.db.ExecContext(ctx, onInvoiceCreated, arg.AddedAt, arg.InvoiceID) + return err +} + +const onInvoiceSettled = `-- name: OnInvoiceSettled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id +) VALUES ( + $1, 2, $2 +) +` + +type OnInvoiceSettledParams struct { + AddedAt time.Time + InvoiceID int64 +} + +func (q *Queries) OnInvoiceSettled(ctx context.Context, arg OnInvoiceSettledParams) error { + _, err := q.db.ExecContext(ctx, onInvoiceSettled, arg.AddedAt, arg.InvoiceID) + return err } diff --git a/sqldb/sqlc/invoices.sql.go b/sqldb/sqlc/invoices.sql.go index fa8d9c294..09ca1a1e2 100644 --- a/sqldb/sqlc/invoices.sql.go +++ b/sqldb/sqlc/invoices.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 // source: invoices.sql package sqlc @@ -11,7 +11,17 @@ import ( "time" ) -const deleteInvoice = `-- name: DeleteInvoice :exec +const deleteCanceledInvoices = `-- name: DeleteCanceledInvoices :execresult +DELETE +FROM invoices +WHERE state = 2 +` + +func (q *Queries) DeleteCanceledInvoices(ctx context.Context) (sql.Result, error) { + return q.db.ExecContext(ctx, deleteCanceledInvoices) +} + +const deleteInvoice = `-- name: DeleteInvoice :execresult DELETE FROM invoices WHERE ( @@ -21,7 +31,7 @@ WHERE ( hash = $2 OR $2 IS NULL ) AND ( - preimage = $3 OR + settle_index = $3 OR $3 IS NULL ) AND ( payment_addr = $4 OR @@ -30,174 +40,24 @@ WHERE ( ` type DeleteInvoiceParams struct { - AddIndex sql.NullInt32 + AddIndex sql.NullInt64 Hash []byte - Preimage []byte + SettleIndex sql.NullInt64 PaymentAddr []byte } -func (q *Queries) DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) error { - _, err := q.db.ExecContext(ctx, deleteInvoice, +func (q *Queries) DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) (sql.Result, error) { + return q.db.ExecContext(ctx, deleteInvoice, arg.AddIndex, arg.Hash, - arg.Preimage, + arg.SettleIndex, arg.PaymentAddr, ) - return err -} - -const deleteInvoiceFeatures = `-- name: DeleteInvoiceFeatures :exec -DELETE -FROM invoice_features -WHERE invoice_id = $1 -` - -func (q *Queries) DeleteInvoiceFeatures(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteInvoiceFeatures, invoiceID) - return err -} - -const deleteInvoiceHTLC = `-- name: DeleteInvoiceHTLC :exec -DELETE -FROM invoice_htlcs -WHERE htlc_id = $1 -` - -func (q *Queries) DeleteInvoiceHTLC(ctx context.Context, htlcID int64) error { - _, err := q.db.ExecContext(ctx, deleteInvoiceHTLC, htlcID) - return err -} - -const deleteInvoiceHTLCCustomRecords = `-- name: DeleteInvoiceHTLCCustomRecords :exec -WITH htlc_ids AS ( - SELECT ih.id - FROM invoice_htlcs ih JOIN invoice_htlc_custom_records ihcr ON ih.id=ihcr.htlc_id - WHERE ih.invoice_id = $1 -) -DELETE -FROM invoice_htlc_custom_records -WHERE htlc_id IN (SELECT id FROM htlc_ids) -` - -func (q *Queries) DeleteInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteInvoiceHTLCCustomRecords, invoiceID) - return err -} - -const deleteInvoiceHTLCs = `-- name: DeleteInvoiceHTLCs :exec -DELETE -FROM invoice_htlcs -WHERE invoice_id = $1 -` - -func (q *Queries) DeleteInvoiceHTLCs(ctx context.Context, invoiceID int32) error { - _, err := q.db.ExecContext(ctx, deleteInvoiceHTLCs, invoiceID) - return err -} - -const filterInvoicePayments = `-- name: FilterInvoicePayments :many -SELECT - ip.id AS settle_index, ip.amount_paid_msat, ip.settled_at AS settle_date, - i.id, i.hash, i.preimage, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at -FROM invoice_payments ip JOIN invoices i ON ip.invoice_id = i.id -WHERE ( - ip.id >= $1 OR - $1 IS NULL -) AND ( - ip.settled_at >= $2 OR - $2 IS NULL -) -ORDER BY - CASE - WHEN $3 = FALSE THEN ip.id - ELSE NULL - END ASC, - CASE - WHEN $3 = TRUE THEN ip.id - ELSE NULL - END DESC -LIMIT $5 OFFSET $4 -` - -type FilterInvoicePaymentsParams struct { - SettleIndexGet sql.NullInt32 - SettledAfter sql.NullTime - Reverse interface{} - NumOffset int32 - NumLimit int32 -} - -type FilterInvoicePaymentsRow struct { - SettleIndex int32 - AmountPaidMsat int64 - SettleDate time.Time - ID int32 - Hash []byte - Preimage []byte - Memo sql.NullString - AmountMsat int64 - CltvDelta sql.NullInt32 - Expiry int32 - PaymentAddr []byte - PaymentRequest sql.NullString - State int16 - AmountPaidMsat_2 int64 - IsAmp bool - IsHodl bool - IsKeysend bool - CreatedAt time.Time -} - -func (q *Queries) FilterInvoicePayments(ctx context.Context, arg FilterInvoicePaymentsParams) ([]FilterInvoicePaymentsRow, error) { - rows, err := q.db.QueryContext(ctx, filterInvoicePayments, - arg.SettleIndexGet, - arg.SettledAfter, - arg.Reverse, - arg.NumOffset, - arg.NumLimit, - ) - if err != nil { - return nil, err - } - defer rows.Close() - var items []FilterInvoicePaymentsRow - for rows.Next() { - var i FilterInvoicePaymentsRow - if err := rows.Scan( - &i.SettleIndex, - &i.AmountPaidMsat, - &i.SettleDate, - &i.ID, - &i.Hash, - &i.Preimage, - &i.Memo, - &i.AmountMsat, - &i.CltvDelta, - &i.Expiry, - &i.PaymentAddr, - &i.PaymentRequest, - &i.State, - &i.AmountPaidMsat_2, - &i.IsAmp, - &i.IsHodl, - &i.IsKeysend, - &i.CreatedAt, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil } const filterInvoices = `-- name: FilterInvoices :many -SELECT id, hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, created_at +SELECT + invoices.id, invoices.hash, invoices.preimage, invoices.settle_index, invoices.settled_at, invoices.memo, invoices.amount_msat, invoices.cltv_delta, invoices.expiry, invoices.payment_addr, invoices.payment_request, invoices.payment_request_hash, invoices.state, invoices.amount_paid_msat, invoices.is_amp, invoices.is_hodl, invoices.is_keysend, invoices.created_at FROM invoices WHERE ( id >= $1 OR @@ -206,48 +66,58 @@ WHERE ( id <= $2 OR $2 IS NULL ) AND ( - state = $3 OR + settle_index >= $3 OR $3 IS NULL ) AND ( - created_at >= $4 OR + settle_index <= $4 OR $4 IS NULL ) AND ( - created_at <= $5 OR + state = $5 OR $5 IS NULL +) AND ( + created_at >= $6 OR + $6 IS NULL +) AND ( + created_at <= $7 OR + $7 IS NULL ) AND ( CASE - WHEN $6=TRUE THEN (state = 0 OR state = 3) + WHEN $8=TRUE THEN (state = 0 OR state = 3) ELSE TRUE END ) ORDER BY CASE - WHEN $7 = FALSE THEN id + WHEN $9 = FALSE THEN id ELSE NULL END ASC, CASE - WHEN $7 = TRUE THEN id + WHEN $9 = TRUE THEN id ELSE NULL END DESC -LIMIT $9 OFFSET $8 +LIMIT $11 OFFSET $10 ` type FilterInvoicesParams struct { - AddIndexGet sql.NullInt32 - AddIndexLet sql.NullInt32 - State sql.NullInt16 - CreatedAfter sql.NullTime - CreatedBefore sql.NullTime - PendingOnly interface{} - Reverse interface{} - NumOffset int32 - NumLimit int32 + AddIndexGet sql.NullInt64 + AddIndexLet sql.NullInt64 + SettleIndexGet sql.NullInt64 + SettleIndexLet sql.NullInt64 + State sql.NullInt16 + CreatedAfter sql.NullTime + CreatedBefore sql.NullTime + PendingOnly interface{} + Reverse interface{} + NumOffset int32 + NumLimit int32 } func (q *Queries) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) ([]Invoice, error) { rows, err := q.db.QueryContext(ctx, filterInvoices, arg.AddIndexGet, arg.AddIndexLet, + arg.SettleIndexGet, + arg.SettleIndexLet, arg.State, arg.CreatedAfter, arg.CreatedBefore, @@ -267,12 +137,15 @@ func (q *Queries) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) &i.ID, &i.Hash, &i.Preimage, + &i.SettleIndex, + &i.SettledAt, &i.Memo, &i.AmountMsat, &i.CltvDelta, &i.Expiry, &i.PaymentAddr, &i.PaymentRequest, + &i.PaymentRequestHash, &i.State, &i.AmountPaidMsat, &i.IsAmp, @@ -295,29 +168,35 @@ func (q *Queries) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) const getInvoice = `-- name: GetInvoice :many -SELECT id, hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, created_at -FROM invoices +SELECT i.id, i.hash, i.preimage, i.settle_index, i.settled_at, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.payment_request_hash, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at +FROM invoices i +LEFT JOIN amp_sub_invoices a on i.id = a.invoice_id WHERE ( - id = $1 OR + i.id = $1 OR $1 IS NULL ) AND ( - hash = $2 OR + i.hash = $2 OR $2 IS NULL ) AND ( - preimage = $3 OR + i.preimage = $3 OR $3 IS NULL ) AND ( - payment_addr = $4 OR + i.payment_addr = $4 OR $4 IS NULL +) AND ( + a.set_id = $5 OR + $5 IS NULL ) +GROUP BY i.id LIMIT 2 ` type GetInvoiceParams struct { - AddIndex sql.NullInt32 + AddIndex sql.NullInt64 Hash []byte Preimage []byte PaymentAddr []byte + SetID []byte } // This method may return more than one invoice if filter using multiple fields @@ -329,6 +208,7 @@ func (q *Queries) GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoi arg.Hash, arg.Preimage, arg.PaymentAddr, + arg.SetID, ) if err != nil { return nil, err @@ -341,12 +221,15 @@ func (q *Queries) GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoi &i.ID, &i.Hash, &i.Preimage, + &i.SettleIndex, + &i.SettledAt, &i.Memo, &i.AmountMsat, &i.CltvDelta, &i.Expiry, &i.PaymentAddr, &i.PaymentRequest, + &i.PaymentRequestHash, &i.State, &i.AmountPaidMsat, &i.IsAmp, @@ -373,7 +256,7 @@ FROM invoice_features WHERE invoice_id = $1 ` -func (q *Queries) GetInvoiceFeatures(ctx context.Context, invoiceID int32) ([]InvoiceFeature, error) { +func (q *Queries) GetInvoiceFeatures(ctx context.Context, invoiceID int64) ([]InvoiceFeature, error) { rows, err := q.db.QueryContext(ctx, getInvoiceFeatures, invoiceID) if err != nil { return nil, err @@ -408,7 +291,7 @@ type GetInvoiceHTLCCustomRecordsRow struct { Value []byte } -func (q *Queries) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) ([]GetInvoiceHTLCCustomRecordsRow, error) { +func (q *Queries) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error) { rows, err := q.db.QueryContext(ctx, getInvoiceHTLCCustomRecords, invoiceID) if err != nil { return nil, err @@ -432,12 +315,12 @@ func (q *Queries) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int } const getInvoiceHTLCs = `-- name: GetInvoiceHTLCs :many -SELECT id, htlc_id, chan_id, amount_msat, total_mpp_msat, accept_height, accept_time, expiry_height, state, resolve_time, invoice_id +SELECT id, chan_id, htlc_id, amount_msat, total_mpp_msat, accept_height, accept_time, expiry_height, state, resolve_time, invoice_id FROM invoice_htlcs WHERE invoice_id = $1 ` -func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]InvoiceHtlc, error) { +func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int64) ([]InvoiceHtlc, error) { rows, err := q.db.QueryContext(ctx, getInvoiceHTLCs, invoiceID) if err != nil { return nil, err @@ -448,8 +331,8 @@ func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]Invoi var i InvoiceHtlc if err := rows.Scan( &i.ID, - &i.HtlcID, &i.ChanID, + &i.HtlcID, &i.AmountMsat, &i.TotalMppMsat, &i.AcceptHeight, @@ -472,68 +355,35 @@ func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]Invoi return items, nil } -const getInvoicePayments = `-- name: GetInvoicePayments :many -SELECT id, settled_at, amount_paid_msat, invoice_id -FROM invoice_payments -WHERE invoice_id = $1 -` - -func (q *Queries) GetInvoicePayments(ctx context.Context, invoiceID int32) ([]InvoicePayment, error) { - rows, err := q.db.QueryContext(ctx, getInvoicePayments, invoiceID) - if err != nil { - return nil, err - } - defer rows.Close() - var items []InvoicePayment - for rows.Next() { - var i InvoicePayment - if err := rows.Scan( - &i.ID, - &i.SettledAt, - &i.AmountPaidMsat, - &i.InvoiceID, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const insertInvoice = `-- name: InsertInvoice :one INSERT INTO invoices ( hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, - payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, - created_at + payment_request, payment_request_hash, state, amount_paid_msat, is_amp, + is_hodl, is_keysend, created_at ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15 ) RETURNING id ` type InsertInvoiceParams struct { - Hash []byte - Preimage []byte - Memo sql.NullString - AmountMsat int64 - CltvDelta sql.NullInt32 - Expiry int32 - PaymentAddr []byte - PaymentRequest sql.NullString - State int16 - AmountPaidMsat int64 - IsAmp bool - IsHodl bool - IsKeysend bool - CreatedAt time.Time + Hash []byte + Preimage []byte + Memo sql.NullString + AmountMsat int64 + CltvDelta sql.NullInt32 + Expiry int32 + PaymentAddr []byte + PaymentRequest sql.NullString + PaymentRequestHash []byte + State int16 + AmountPaidMsat int64 + IsAmp bool + IsHodl bool + IsKeysend bool + CreatedAt time.Time } -func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int32, error) { +func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int64, error) { row := q.db.QueryRowContext(ctx, insertInvoice, arg.Hash, arg.Preimage, @@ -543,6 +393,7 @@ func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (i arg.Expiry, arg.PaymentAddr, arg.PaymentRequest, + arg.PaymentRequestHash, arg.State, arg.AmountPaidMsat, arg.IsAmp, @@ -550,7 +401,7 @@ func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (i arg.IsKeysend, arg.CreatedAt, ) - var id int32 + var id int64 err := row.Scan(&id) return id, err } @@ -564,7 +415,7 @@ INSERT INTO invoice_features ( ` type InsertInvoiceFeatureParams struct { - InvoiceID int32 + InvoiceID int64 Feature int32 } @@ -573,13 +424,13 @@ func (q *Queries) InsertInvoiceFeature(ctx context.Context, arg InsertInvoiceFea return err } -const insertInvoiceHTLC = `-- name: InsertInvoiceHTLC :exec +const insertInvoiceHTLC = `-- name: InsertInvoiceHTLC :one INSERT INTO invoice_htlcs ( htlc_id, chan_id, amount_msat, total_mpp_msat, accept_height, accept_time, expiry_height, state, resolve_time, invoice_id ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 -) +) RETURNING id ` type InsertInvoiceHTLCParams struct { @@ -592,11 +443,11 @@ type InsertInvoiceHTLCParams struct { ExpiryHeight int32 State int16 ResolveTime sql.NullTime - InvoiceID int32 + InvoiceID int64 } -func (q *Queries) InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) error { - _, err := q.db.ExecContext(ctx, insertInvoiceHTLC, +func (q *Queries) InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertInvoiceHTLC, arg.HtlcID, arg.ChanID, arg.AmountMsat, @@ -608,7 +459,9 @@ func (q *Queries) InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCPa arg.ResolveTime, arg.InvoiceID, ) - return err + var id int64 + err := row.Scan(&id) + return id, err } const insertInvoiceHTLCCustomRecord = `-- name: InsertInvoiceHTLCCustomRecord :exec @@ -630,64 +483,56 @@ func (q *Queries) InsertInvoiceHTLCCustomRecord(ctx context.Context, arg InsertI return err } -const insertInvoicePayment = `-- name: InsertInvoicePayment :one -INSERT INTO invoice_payments ( - invoice_id, amount_paid_msat, settled_at -) VALUES ( - $1, $2, $3 -) RETURNING id +const nextInvoiceSettleIndex = `-- name: NextInvoiceSettleIndex :one +UPDATE invoice_sequences SET current_value = current_value + 1 +WHERE name = 'settle_index' +RETURNING current_value ` -type InsertInvoicePaymentParams struct { - InvoiceID int32 - AmountPaidMsat int64 - SettledAt time.Time +func (q *Queries) NextInvoiceSettleIndex(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, nextInvoiceSettleIndex) + var current_value int64 + err := row.Scan(¤t_value) + return current_value, err } -func (q *Queries) InsertInvoicePayment(ctx context.Context, arg InsertInvoicePaymentParams) (int32, error) { - row := q.db.QueryRowContext(ctx, insertInvoicePayment, arg.InvoiceID, arg.AmountPaidMsat, arg.SettledAt) - var id int32 - err := row.Scan(&id) - return id, err -} - -const updateInvoice = `-- name: UpdateInvoice :exec -UPDATE invoices -SET preimage=$2, state=$3, amount_paid_msat=$4 -WHERE id=$1 +const updateInvoiceAmountPaid = `-- name: UpdateInvoiceAmountPaid :execresult +UPDATE invoices +SET amount_paid_msat = $2 +WHERE id = $1 ` -type UpdateInvoiceParams struct { - ID int32 - Preimage []byte - State int16 +type UpdateInvoiceAmountPaidParams struct { + ID int64 AmountPaidMsat int64 } -func (q *Queries) UpdateInvoice(ctx context.Context, arg UpdateInvoiceParams) error { - _, err := q.db.ExecContext(ctx, updateInvoice, - arg.ID, - arg.Preimage, - arg.State, - arg.AmountPaidMsat, - ) - return err +func (q *Queries) UpdateInvoiceAmountPaid(ctx context.Context, arg UpdateInvoiceAmountPaidParams) (sql.Result, error) { + return q.db.ExecContext(ctx, updateInvoiceAmountPaid, arg.ID, arg.AmountPaidMsat) } const updateInvoiceHTLC = `-- name: UpdateInvoiceHTLC :exec UPDATE invoice_htlcs -SET state=$2, resolve_time=$3 -WHERE id = $1 +SET state=$4, resolve_time=$5 +WHERE htlc_id = $1 AND chan_id = $2 AND invoice_id = $3 ` type UpdateInvoiceHTLCParams struct { - ID int32 + HtlcID int64 + ChanID string + InvoiceID int64 State int16 ResolveTime sql.NullTime } func (q *Queries) UpdateInvoiceHTLC(ctx context.Context, arg UpdateInvoiceHTLCParams) error { - _, err := q.db.ExecContext(ctx, updateInvoiceHTLC, arg.ID, arg.State, arg.ResolveTime) + _, err := q.db.ExecContext(ctx, updateInvoiceHTLC, + arg.HtlcID, + arg.ChanID, + arg.InvoiceID, + arg.State, + arg.ResolveTime, + ) return err } @@ -698,7 +543,7 @@ WHERE invoice_id = $1 AND resolve_time IS NULL ` type UpdateInvoiceHTLCsParams struct { - InvoiceID int32 + InvoiceID int64 State int16 ResolveTime sql.NullTime } @@ -707,3 +552,30 @@ func (q *Queries) UpdateInvoiceHTLCs(ctx context.Context, arg UpdateInvoiceHTLCs _, err := q.db.ExecContext(ctx, updateInvoiceHTLCs, arg.InvoiceID, arg.State, arg.ResolveTime) return err } + +const updateInvoiceState = `-- name: UpdateInvoiceState :execresult +UPDATE invoices +SET state = $2, + preimage = COALESCE(preimage, $3), + settle_index = COALESCE(settle_index, $4), + settled_at = COALESCE(settled_at, $5) +WHERE id = $1 +` + +type UpdateInvoiceStateParams struct { + ID int64 + State int16 + Preimage []byte + SettleIndex sql.NullInt64 + SettledAt sql.NullTime +} + +func (q *Queries) UpdateInvoiceState(ctx context.Context, arg UpdateInvoiceStateParams) (sql.Result, error) { + return q.db.ExecContext(ctx, updateInvoiceState, + arg.ID, + arg.State, + arg.Preimage, + arg.SettleIndex, + arg.SettledAt, + ) +} diff --git a/sqldb/sqlc/migrations/000001_invoices.up.sql b/sqldb/sqlc/migrations/000001_invoices.up.sql index 32b70b9fa..e4c80a26f 100644 --- a/sqldb/sqlc/migrations/000001_invoices.up.sql +++ b/sqldb/sqlc/migrations/000001_invoices.up.sql @@ -1,7 +1,17 @@ +-- sequences contains all sequences used for invoices. +CREATE TABLE invoice_sequences ( + name TEXT PRIMARY KEY, + current_value BIGINT NOT NULL +); + +-- Initialize a sequence for the settled index used to track invoice settlement +-- to remain compatible with the legacy channeldb implementation. +INSERT INTO invoice_sequences(name, current_value) VALUES ('settle_index', 0); + -- invoices table contains all the information shared by all the invoice types. CREATE TABLE IF NOT EXISTS invoices ( -- The id of the invoice. Translates to the AddIndex. - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, -- The hash for this invoice. The invoice hash will always identify that -- invoice. @@ -10,6 +20,14 @@ CREATE TABLE IF NOT EXISTS invoices ( -- The preimage for the hash in this invoice. Some invoices may have this -- field empty, like unsettled hodl invoices or AMP invoices. preimage BLOB, + + -- If settled, the index is set to the current_value+1 of the settle_index + -- seuqence in the invoice_sequences table. If not settled, then it is set + -- to NULL. + settle_index BIGINT, + + -- When the invoice was settled. + settled_at TIMESTAMP, -- An optional memo to attach along with the invoice. memo TEXT, @@ -31,7 +49,12 @@ CREATE TABLE IF NOT EXISTS invoices ( -- The encoded payment request for this invoice. Some invoice types may -- not have leave this empty, like Keysends. - payment_request TEXT UNIQUE, + payment_request TEXT, + + -- Holds the hash of the payment request. This field is used to ensure that + -- there are no duplicates in the database. This trick is needed because + -- PostgreSQL has a limitiation of 2712 bytes for unique index rows. + payment_request_hash BLOB UNIQUE, -- The invoice state. state SMALLINT NOT NULL, @@ -58,6 +81,7 @@ CREATE INDEX IF NOT EXISTS invoices_preimage_idx ON invoices(preimage); CREATE INDEX IF NOT EXISTS invoices_payment_addr_idx ON invoices(payment_addr); CREATE INDEX IF NOT EXISTS invoices_state_idx ON invoices(state); CREATE INDEX IF NOT EXISTS invoices_created_at_idx ON invoices(created_at); +CREATE INDEX IF NOT EXISTS invoices_settled_at_idx ON invoices(settled_at); -- invoice_features contains the feature bits of an invoice. CREATE TABLE IF NOT EXISTS invoice_features ( @@ -65,7 +89,7 @@ CREATE TABLE IF NOT EXISTS invoice_features ( feature INTEGER NOT NULL, -- The invoice id this feature belongs to. - invoice_id INTEGER NOT NULL REFERENCES invoices(id), + invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, -- The feature bit is unique per invoice. UNIQUE (feature, invoice_id) @@ -78,15 +102,15 @@ CREATE INDEX IF NOT EXISTS invoice_feature_invoice_id_idx ON invoice_features(in CREATE TABLE IF NOT EXISTS invoice_htlcs ( -- The id for this htlc. Used in foreign keys instead of the -- htlc_id/chan_id combination. - id INTEGER PRIMARY KEY, + id BIGINT PRIMARY KEY, + + -- Short chan id indicating the htlc's origin. uint64 stored as text. + chan_id TEXT NOT NULL, -- The uint64 htlc id. This field is a counter so it is safe to store it as -- int64 in the database. The application layer must check that there is no -- overflow when storing/loading this column. htlc_id BIGINT NOT NULL, - - -- Short chan id indicating the htlc's origin. uint64 stored as text. - chan_id TEXT NOT NULL, -- The htlc's amount in millisatoshis. amount_msat BIGINT NOT NULL, @@ -110,10 +134,10 @@ CREATE TABLE IF NOT EXISTS invoice_htlcs ( resolve_time TIMESTAMP, -- The id of the invoice this htlc belongs to. - invoice_id INTEGER NOT NULL REFERENCES invoices(id), + invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, -- The htlc_id and chan_id identify the htlc. - UNIQUE (htlc_id, chan_id) + UNIQUE (chan_id, htlc_id) ); CREATE INDEX IF NOT EXISTS invoice_htlc_invoice_id_idx ON invoice_htlcs(invoice_id); @@ -129,26 +153,8 @@ CREATE TABLE IF NOT EXISTS invoice_htlc_custom_records ( value BLOB NOT NULL, -- The htlc id this record belongs to. - htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id) + htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS invoice_htlc_custom_records_htlc_id_idx ON invoice_htlc_custom_records(htlc_id); --- invoice_payments contains the information of a settled invoice payment. -CREATE TABLE IF NOT EXISTS invoice_payments ( - -- The id for this invoice payment. Translates to SettleIndex. - id INTEGER PRIMARY KEY, - - -- When the payment was settled. - settled_at TIMESTAMP NOT NULL, - - -- The amount of the payment in millisatoshis. This is the sum of all the - -- the htlcs settled for this payment. - amount_paid_msat BIGINT NOT NULL, - - -- The invoice id this payment is for. - invoice_id INTEGER NOT NULL REFERENCES invoices(id) -); - -CREATE INDEX IF NOT EXISTS invoice_payments_settled_at_idx ON invoice_payments(settled_at); -CREATE INDEX IF NOT EXISTS invoice_payments_invoice_id_idx ON invoice_payments(invoice_id); diff --git a/sqldb/sqlc/migrations/000002_amp_invoices.up.sql b/sqldb/sqlc/migrations/000002_amp_invoices.up.sql index a4daa8c30..7204e775b 100644 --- a/sqldb/sqlc/migrations/000002_amp_invoices.up.sql +++ b/sqldb/sqlc/migrations/000002_amp_invoices.up.sql @@ -1,5 +1,6 @@ --- amp_invoices_payments -CREATE TABLE IF NOT EXISTS amp_invoice_payments ( +-- amp_sub_invoices holds all AMP sub-invoices that belong to a given parent +-- invoice. +CREATE TABLE IF NOT EXISTS amp_sub_invoices ( -- The set id identifying the payment. set_id BLOB PRIMARY KEY, @@ -10,26 +11,35 @@ CREATE TABLE IF NOT EXISTS amp_invoice_payments ( -- Timestamp of when the first htlc for this payment was accepted. created_at TIMESTAMP NOT NULL, - -- If settled, the invoice payment related to this set id. - settled_index INTEGER REFERENCES invoice_payments(id), + -- When the invoice was settled. + settled_at TIMESTAMP, - -- The invoice id this set id is related to. - invoice_id INTEGER NOT NULL REFERENCES invoices(id) + -- If settled, the index is set to the current_value+1 of the settle_index + -- seuqence in the invoice_sequences table. If not settled, then it is set + -- to NULL. + settle_index BIGINT, + + -- The id of the parent invoice this AMP invoice belongs to. + invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, + + -- A unique constraint on the set_id and invoice_id is needed to ensure that + -- we don't have two AMP invoices for the same invoice. + UNIQUE (set_id, invoice_id) ); -CREATE INDEX IF NOT EXISTS amp_invoice_payments_invoice_id_idx ON amp_invoice_payments(invoice_id); +CREATE INDEX IF NOT EXISTS amp_sub_invoices_invoice_id_idx ON amp_sub_invoices(invoice_id); -- amp_invoice_htlcs contains the complementary information for an htlc related -- to an AMP invoice. -CREATE TABLE IF NOT EXISTS amp_invoice_htlcs ( +CREATE TABLE IF NOT EXISTS amp_sub_invoice_htlcs ( + -- The invoice id this entry is related to. + invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, + -- The set id identifying the payment this htlc belongs to. - set_id BLOB NOT NULL REFERENCES amp_invoice_payments(set_id), + set_id BLOB NOT NULL REFERENCES amp_sub_invoices(set_id) ON DELETE CASCADE, -- The id of the htlc this entry blongs to. - htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id), - - -- The invoice id this entry is related to. - invoice_id INTEGER NOT NULL REFERENCES invoices(id), + htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id) ON DELETE CASCADE, -- The root share for this amp htlc. root_share BLOB NOT NULL, @@ -48,7 +58,7 @@ CREATE TABLE IF NOT EXISTS amp_invoice_htlcs ( preimage BLOB ); -CREATE INDEX IF NOT EXISTS amp_htlcs_set_id_idx ON amp_invoice_htlcs(set_id); -CREATE INDEX IF NOT EXISTS amp_htlcs_invoice_id_idx ON amp_invoice_htlcs(invoice_id); -CREATE INDEX IF NOT EXISTS amp_htlcs_htlc_id_idx ON amp_invoice_htlcs(htlc_id); +CREATE INDEX IF NOT EXISTS amp_htlcs_invoice_id_idx ON amp_sub_invoice_htlcs(invoice_id); +CREATE INDEX IF NOT EXISTS amp_htlcs_set_id_idx ON amp_sub_invoice_htlcs(set_id); +CREATE INDEX IF NOT EXISTS amp_htlcs_htlc_id_idx ON amp_sub_invoice_htlcs(htlc_id); diff --git a/sqldb/sqlc/migrations/000003_invoice_events.down.sql b/sqldb/sqlc/migrations/000003_invoice_events.down.sql index 66e554e24..3f447dfa8 100644 --- a/sqldb/sqlc/migrations/000003_invoice_events.down.sql +++ b/sqldb/sqlc/migrations/000003_invoice_events.down.sql @@ -1,9 +1,6 @@ DROP TABLE IF EXISTS invoice_events; - -DROP INDEX IF EXISTS invoice_events_created_at_idx; -DROP INDEX IF EXISTS invoice_events_invoice_id_idx; -DROP INDEX IF EXISTS invoice_events_htlc_id_idx; -DROP INDEX IF EXISTS invoice_events_set_id_idx; +DROP INDEX IF EXISTS invoice_events_added_at_idx; DROP INDEX IF EXISTS invoice_events_event_type_idx; - +DROP INDEX IF EXISTS invoice_events_invoice_id_idx; +DROP INDEX IF EXISTS invoice_events_set_id_idx; DROP TABLE IF EXISTS invoice_event_types; diff --git a/sqldb/sqlc/migrations/000003_invoice_events.up.sql b/sqldb/sqlc/migrations/000003_invoice_events.up.sql index 47d2a91d0..a3718c0eb 100644 --- a/sqldb/sqlc/migrations/000003_invoice_events.up.sql +++ b/sqldb/sqlc/migrations/000003_invoice_events.up.sql @@ -1,59 +1,51 @@ --- invoice_event_types stores the different types of events that can be emitted --- for invoices. +-- invoice_event_types stores the different types of invoice events. CREATE TABLE IF NOT EXISTS invoice_event_types( id INTEGER PRIMARY KEY, description TEXT NOT NULL ); --- invoice_events stores all events related to the node invoices. -CREATE TABLE IF NOT EXISTS invoice_events ( - id INTEGER PRIMARY KEY, - - -- created_at is the creation time of this event. - created_at TIMESTAMP NOT NULL, - - -- invoice_id is the reference to the invoice this event was emitted for. - invoice_id INTEGER NOT NULL REFERENCES invoices(id), - - -- htlc_id is the reference to the htlc this event was emitted for, may be - -- null. - htlc_id BIGINT REFERENCES invoice_htlcs(htlc_id), - - -- set_id is the reference to the set_id this event was emitted for, may be - -- null. - set_id BLOB NOT NULL REFERENCES amp_invoice_payments(set_id), - - -- event_type is the type of this event. - event_type INTEGER NOT NULL REFERENCES invoice_event_types(id), - - -- event_metadata is a TLV encoding any relevant information for this kind - -- of events. - event_metadata BLOB -); - -CREATE INDEX IF NOT EXISTS invoice_events_created_at_idx ON invoice_events(created_at); -CREATE INDEX IF NOT EXISTS invoice_events_invoice_id_idx ON invoice_events(invoice_id); -CREATE INDEX IF NOT EXISTS invoice_events_htlc_id_idx ON invoice_events(htlc_id); -CREATE INDEX IF NOT EXISTS invoice_events_set_id_idx ON invoice_events(set_id); -CREATE INDEX IF NOT EXISTS invoice_events_event_type_idx ON invoice_events(event_type); - - --- invoice_event_types defines the different types of events that can be emitted --- for an invoice. +-- invoice_event_types defines the different types of invoice events. INSERT INTO invoice_event_types (id, description) VALUES - -- invoice_created is the event emitted when an invoice is created. + -- invoice_created is the event type used when an invoice is created. (0, 'invoice_created'), - -- invoice_canceled is the event emitted when an invoice is canceled. - (1, "invoice_canceled"), - -- invoice_settled is the event emitted when an invoice is settled. - (2, "invoice_settled"), - -- setid_created is the event emitted when the first htlc for the set_id is - -- received. - (3, "setid_created"), - -- setid_canceled is the event emitted when the set_id is canceled. - (4, "setid_canceled"), - -- setid_settled is the event emitted when the set_id is settled. - (5, "setid_settled"); + -- invoice_canceled is the event type used when an invoice is canceled. + (1, 'invoice_canceled'), + -- invoice_settled is the event type used when an invoice is settled. + (2, 'invoice_settled'), + -- setid_created is the event type used when an AMP sub invoice + -- corresponding to the set_id is created. + (3, 'setid_created'), + -- setid_canceled is the event type used when an AMP sub invoice + -- corresponding to the set_id is canceled. + (4, 'setid_canceled'), + -- setid_settled is the event type used when an AMP sub invoice + -- corresponding to the set_id is settled. + (5, 'setid_settled'); + +-- invoice_events stores all major events related to the node's invoices and +-- AMP sub invoices. This table can be used to create a historical view of what +-- happened to the node's invoices. +CREATE TABLE IF NOT EXISTS invoice_events ( + id BIGINT PRIMARY KEY, + + -- added_at is the timestamp when this event was added. + added_at TIMESTAMP NOT NULL, + + -- event_type is the type of this event. + event_type INTEGER NOT NULL REFERENCES invoice_event_types(id) ON DELETE CASCADE, + + -- invoice_id is the reference to the invoice this event was added for. + invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, + + -- set_id is the reference to the AMP sub invoice this event was added for. + -- May be NULL if the event is not related to an AMP sub invoice. + set_id BLOB REFERENCES amp_sub_invoices(set_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS invoice_events_added_at_idx ON invoice_events(added_at); +CREATE INDEX IF NOT EXISTS invoice_events_event_type_idx ON invoice_events(event_type); +CREATE INDEX IF NOT EXISTS invoice_events_invoice_id_idx ON invoice_events(invoice_id); +CREATE INDEX IF NOT EXISTS invoice_events_set_id_idx ON invoice_events(set_id); diff --git a/sqldb/sqlc/models.go b/sqldb/sqlc/models.go index b0ef3c857..83be5a708 100644 --- a/sqldb/sqlc/models.go +++ b/sqldb/sqlc/models.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 package sqlc @@ -9,50 +9,52 @@ import ( "time" ) -type AmpInvoiceHtlc struct { +type AmpSubInvoice struct { + SetID []byte + State int16 + CreatedAt time.Time + SettledAt sql.NullTime + SettleIndex sql.NullInt64 + InvoiceID int64 +} + +type AmpSubInvoiceHtlc struct { + InvoiceID int64 SetID []byte HtlcID int64 - InvoiceID int32 RootShare []byte ChildIndex int64 Hash []byte Preimage []byte } -type AmpInvoicePayment struct { - SetID []byte - State int16 - CreatedAt time.Time - SettledIndex sql.NullInt32 - InvoiceID int32 -} - type Invoice struct { - ID int32 - Hash []byte - Preimage []byte - Memo sql.NullString - AmountMsat int64 - CltvDelta sql.NullInt32 - Expiry int32 - PaymentAddr []byte - PaymentRequest sql.NullString - State int16 - AmountPaidMsat int64 - IsAmp bool - IsHodl bool - IsKeysend bool - CreatedAt time.Time + ID int64 + Hash []byte + Preimage []byte + SettleIndex sql.NullInt64 + SettledAt sql.NullTime + Memo sql.NullString + AmountMsat int64 + CltvDelta sql.NullInt32 + Expiry int32 + PaymentAddr []byte + PaymentRequest sql.NullString + PaymentRequestHash []byte + State int16 + AmountPaidMsat int64 + IsAmp bool + IsHodl bool + IsKeysend bool + CreatedAt time.Time } type InvoiceEvent struct { - ID int32 - CreatedAt time.Time - InvoiceID int32 - HtlcID sql.NullInt64 - SetID []byte - EventType int32 - EventMetadata []byte + ID int64 + AddedAt time.Time + EventType int32 + InvoiceID int64 + SetID []byte } type InvoiceEventType struct { @@ -62,13 +64,13 @@ type InvoiceEventType struct { type InvoiceFeature struct { Feature int32 - InvoiceID int32 + InvoiceID int64 } type InvoiceHtlc struct { - ID int32 - HtlcID int64 + ID int64 ChanID string + HtlcID int64 AmountMsat int64 TotalMppMsat sql.NullInt64 AcceptHeight int32 @@ -76,7 +78,7 @@ type InvoiceHtlc struct { ExpiryHeight int32 State int16 ResolveTime sql.NullTime - InvoiceID int32 + InvoiceID int64 } type InvoiceHtlcCustomRecord struct { @@ -85,9 +87,7 @@ type InvoiceHtlcCustomRecord struct { HtlcID int64 } -type InvoicePayment struct { - ID int32 - SettledAt time.Time - AmountPaidMsat int64 - InvoiceID int32 +type InvoiceSequence struct { + Name string + CurrentValue int64 } diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 1e766ac63..d55d8090a 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -1,51 +1,48 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.18.0 +// sqlc v1.25.0 package sqlc import ( "context" + "database/sql" ) type Querier interface { - DeleteAMPHTLCCustomRecords(ctx context.Context, invoiceID int32) error - DeleteAMPHTLCs(ctx context.Context, invoiceID int32) error - DeleteAMPInvoiceHTLC(ctx context.Context, setID []byte) error - DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) error - DeleteInvoiceEvents(ctx context.Context, invoiceID int32) error - DeleteInvoiceFeatures(ctx context.Context, invoiceID int32) error - DeleteInvoiceHTLC(ctx context.Context, htlcID int64) error - DeleteInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) error - DeleteInvoiceHTLCs(ctx context.Context, invoiceID int32) error - FilterInvoicePayments(ctx context.Context, arg FilterInvoicePaymentsParams) ([]FilterInvoicePaymentsRow, error) + DeleteCanceledInvoices(ctx context.Context) (sql.Result, error) + DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) (sql.Result, error) + FetchAMPSubInvoiceHTLCs(ctx context.Context, arg FetchAMPSubInvoiceHTLCsParams) ([]FetchAMPSubInvoiceHTLCsRow, error) + FetchAMPSubInvoices(ctx context.Context, arg FetchAMPSubInvoicesParams) ([]AmpSubInvoice, error) + FetchSettledAMPSubInvoices(ctx context.Context, arg FetchSettledAMPSubInvoicesParams) ([]FetchSettledAMPSubInvoicesRow, error) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) ([]Invoice, error) - GetAMPInvoiceHTLCsByInvoiceID(ctx context.Context, invoiceID int32) ([]AmpInvoiceHtlc, error) - GetAMPInvoiceHTLCsBySetID(ctx context.Context, setID []byte) ([]AmpInvoiceHtlc, error) + GetAMPInvoiceID(ctx context.Context, setID []byte) (int64, error) // This method may return more than one invoice if filter using multiple fields // from different invoices. It is the caller's responsibility to ensure that // we bubble up an error in those cases. GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoice, error) - GetInvoiceFeatures(ctx context.Context, invoiceID int32) ([]InvoiceFeature, error) - GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) ([]GetInvoiceHTLCCustomRecordsRow, error) - GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]InvoiceHtlc, error) - GetInvoicePayments(ctx context.Context, invoiceID int32) ([]InvoicePayment, error) - GetSetIDHTLCsCustomRecords(ctx context.Context, setID []byte) ([]GetSetIDHTLCsCustomRecordsRow, error) - InsertAMPInvoiceHTLC(ctx context.Context, arg InsertAMPInvoiceHTLCParams) error - InsertAMPInvoicePayment(ctx context.Context, arg InsertAMPInvoicePaymentParams) error - InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int32, error) - InsertInvoiceEvent(ctx context.Context, arg InsertInvoiceEventParams) error + GetInvoiceFeatures(ctx context.Context, invoiceID int64) ([]InvoiceFeature, error) + GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error) + GetInvoiceHTLCs(ctx context.Context, invoiceID int64) ([]InvoiceHtlc, error) + InsertAMPSubInvoiceHTLC(ctx context.Context, arg InsertAMPSubInvoiceHTLCParams) error + InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int64, error) InsertInvoiceFeature(ctx context.Context, arg InsertInvoiceFeatureParams) error - InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) error + InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) (int64, error) InsertInvoiceHTLCCustomRecord(ctx context.Context, arg InsertInvoiceHTLCCustomRecordParams) error - InsertInvoicePayment(ctx context.Context, arg InsertInvoicePaymentParams) (int32, error) - SelectAMPInvoicePayments(ctx context.Context, arg SelectAMPInvoicePaymentsParams) ([]SelectAMPInvoicePaymentsRow, error) - SelectInvoiceEvents(ctx context.Context, arg SelectInvoiceEventsParams) ([]InvoiceEvent, error) - UpdateAMPInvoiceHTLC(ctx context.Context, arg UpdateAMPInvoiceHTLCParams) error - UpdateAMPPayment(ctx context.Context, arg UpdateAMPPaymentParams) error - UpdateInvoice(ctx context.Context, arg UpdateInvoiceParams) error + NextInvoiceSettleIndex(ctx context.Context) (int64, error) + OnAMPSubInvoiceCanceled(ctx context.Context, arg OnAMPSubInvoiceCanceledParams) error + OnAMPSubInvoiceCreated(ctx context.Context, arg OnAMPSubInvoiceCreatedParams) error + OnAMPSubInvoiceSettled(ctx context.Context, arg OnAMPSubInvoiceSettledParams) error + OnInvoiceCanceled(ctx context.Context, arg OnInvoiceCanceledParams) error + OnInvoiceCreated(ctx context.Context, arg OnInvoiceCreatedParams) error + OnInvoiceSettled(ctx context.Context, arg OnInvoiceSettledParams) error + UpdateAMPSubInvoiceHTLCPreimage(ctx context.Context, arg UpdateAMPSubInvoiceHTLCPreimageParams) (sql.Result, error) + UpdateAMPSubInvoiceState(ctx context.Context, arg UpdateAMPSubInvoiceStateParams) error + UpdateInvoiceAmountPaid(ctx context.Context, arg UpdateInvoiceAmountPaidParams) (sql.Result, error) UpdateInvoiceHTLC(ctx context.Context, arg UpdateInvoiceHTLCParams) error UpdateInvoiceHTLCs(ctx context.Context, arg UpdateInvoiceHTLCsParams) error + UpdateInvoiceState(ctx context.Context, arg UpdateInvoiceStateParams) (sql.Result, error) + UpsertAMPSubInvoice(ctx context.Context, arg UpsertAMPSubInvoiceParams) (sql.Result, error) } var _ Querier = (*Queries)(nil) diff --git a/sqldb/sqlc/queries/amp_invoices.sql b/sqldb/sqlc/queries/amp_invoices.sql index d965de768..3b6ee76ac 100644 --- a/sqldb/sqlc/queries/amp_invoices.sql +++ b/sqldb/sqlc/queries/amp_invoices.sql @@ -1,79 +1,67 @@ --- name: InsertAMPInvoicePayment :exec -INSERT INTO amp_invoice_payments ( - set_id, state, created_at, settled_index, invoice_id +-- name: UpsertAMPSubInvoice :execresult +INSERT INTO amp_sub_invoices ( + set_id, state, created_at, invoice_id ) VALUES ( - $1, $2, $3, $4, $5 -); + $1, $2, $3, $4 +) ON CONFLICT (set_id, invoice_id) DO NOTHING; --- name: SelectAMPInvoicePayments :many -SELECT aip.*, ip.* -FROM amp_invoice_payments aip LEFT JOIN invoice_payments ip ON aip.settled_index = ip.id -WHERE ( - set_id = sqlc.narg('set_id') OR - sqlc.narg('set_id') IS NULL -) AND ( - aip.settled_index = sqlc.narg('settled_index') OR - sqlc.narg('settled_index') IS NULL -) AND ( - aip.invoice_id = sqlc.narg('invoice_id') OR - sqlc.narg('invoice_id') IS NULL -); +-- name: GetAMPInvoiceID :one +SELECT invoice_id FROM amp_sub_invoices WHERE set_id = $1; --- name: UpdateAMPPayment :exec -UPDATE amp_invoice_payments -SET state = $1, settled_index = $2 -WHERE state = 0 AND ( - set_id = sqlc.narg('set_id') OR - sqlc.narg('set_id') IS NULL -) AND ( - invoice_id = sqlc.narg('invoice_id') OR - sqlc.narg('invoice_id') IS NULL -); +-- name: UpdateAMPSubInvoiceState :exec +UPDATE amp_sub_invoices +SET state = $2, + settle_index = COALESCE(settle_index, $3), + settled_at = COALESCE(settled_at, $4) +WHERE set_id = $1; --- name: InsertAMPInvoiceHTLC :exec -INSERT INTO amp_invoice_htlcs ( - set_id, htlc_id, root_share, child_index, hash, preimage +-- name: InsertAMPSubInvoiceHTLC :exec +INSERT INTO amp_sub_invoice_htlcs ( + invoice_id, set_id, htlc_id, root_share, child_index, hash, preimage ) VALUES ( - $1, $2, $3, $4, $5, $6 + $1, $2, $3, $4, $5, $6, $7 ); --- name: GetAMPInvoiceHTLCsBySetID :many +-- name: FetchAMPSubInvoices :many SELECT * -FROM amp_invoice_htlcs -WHERE set_id = $1; +FROM amp_sub_invoices +WHERE invoice_id = $1 +AND ( + set_id = sqlc.narg('set_id') OR + sqlc.narg('set_id') IS NULL +); --- name: GetAMPInvoiceHTLCsByInvoiceID :many -SELECT * -FROM amp_invoice_htlcs -WHERE invoice_id = $1; +-- name: FetchAMPSubInvoiceHTLCs :many +SELECT + amp.set_id, amp.root_share, amp.child_index, amp.hash, amp.preimage, + invoice_htlcs.* +FROM amp_sub_invoice_htlcs amp +INNER JOIN invoice_htlcs ON amp.htlc_id = invoice_htlcs.id +WHERE amp.invoice_id = $1 +AND ( + set_id = sqlc.narg('set_id') OR + sqlc.narg('set_id') IS NULL +); --- name: GetSetIDHTLCsCustomRecords :many -SELECT ihcr.htlc_id, key, value -FROM amp_invoice_htlcs aih JOIN invoice_htlc_custom_records ihcr ON aih.id=ihcr.htlc_id -WHERE aih.set_id = $1; - --- name: UpdateAMPInvoiceHTLC :exec -UPDATE amp_invoice_htlcs -SET preimage = $1 -WHERE htlc_id = $2; - --- name: DeleteAMPHTLCCustomRecords :exec -WITH htlc_ids AS ( - SELECT htlc_id - FROM amp_invoice_htlcs - WHERE invoice_id = $1 -) -DELETE -FROM invoice_htlc_custom_records -WHERE htlc_id IN (SELECT id FROM htlc_ids); - --- name: DeleteAMPHTLCs :exec -DELETE -FROM amp_invoice_htlcs -WHERE invoice_id = $1; - --- name: DeleteAMPInvoiceHTLC :exec -DELETE -FROM amp_invoice_htlcs -WHERE set_id = $1; +-- name: FetchSettledAMPSubInvoices :many +SELECT + a.set_id, + a.settle_index as amp_settle_index, + a.settled_at as amp_settled_at, + i.* +FROM amp_sub_invoices a +INNER JOIN invoices i ON a.invoice_id = i.id +WHERE ( + a.settle_index >= sqlc.narg('settle_index_get') OR + sqlc.narg('settle_index_get') IS NULL +) AND ( + a.settle_index <= sqlc.narg('settle_index_let') OR + sqlc.narg('settle_index_let') IS NULL +); +-- name: UpdateAMPSubInvoiceHTLCPreimage :execresult +UPDATE amp_sub_invoice_htlcs AS a +SET preimage = $4 +WHERE a.invoice_id = $1 AND a.set_id = $2 AND a.htlc_id = ( + SELECT id FROM invoice_htlcs AS i WHERE i.htlc_id = $3 +); diff --git a/sqldb/sqlc/queries/invoice_events.sql b/sqldb/sqlc/queries/invoice_events.sql index 4216a2513..4192c466b 100644 --- a/sqldb/sqlc/queries/invoice_events.sql +++ b/sqldb/sqlc/queries/invoice_events.sql @@ -1,35 +1,41 @@ --- name: InsertInvoiceEvent :exec +-- name: OnInvoiceCreated :exec INSERT INTO invoice_events ( - created_at, invoice_id, htlc_id, set_id, event_type, event_metadata + added_at, event_type, invoice_id ) VALUES ( - $1, $2, $3, $4, $5, $6 + $1, 0, $2 ); --- name: SelectInvoiceEvents :many -SELECT * -FROM invoice_events -WHERE ( - invoice_id = sqlc.narg('invoice_id') OR - sqlc.narg('invoice_id') IS NULL -) AND ( - htlc_id = sqlc.narg('htlc_id') OR - sqlc.narg('htlc_id') IS NULL -) AND ( - set_id = sqlc.narg('set_id') OR - sqlc.narg('set_id') IS NULL -) AND ( - event_type = sqlc.narg('event_type') OR - sqlc.narg('event_type') IS NULL -) AND ( - created_at >= sqlc.narg('created_after') OR - sqlc.narg('created_after') IS NULL -) AND ( - created_at <= sqlc.narg('created_before') OR - sqlc.narg('created_before') IS NULL -) -LIMIT @num_limit OFFSET @num_offset; +-- name: OnInvoiceCanceled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id +) VALUES ( + $1, 1, $2 +); --- name: DeleteInvoiceEvents :exec -DELETE -FROM invoice_events -WHERE invoice_id = $1; +-- name: OnInvoiceSettled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id +) VALUES ( + $1, 2, $2 +); + +-- name: OnAMPSubInvoiceCreated :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id, set_id +) VALUES ( + $1, 3, $2, $3 +); + +-- name: OnAMPSubInvoiceCanceled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id, set_id +) VALUES ( + $1, 4, $2, $3 +); + +-- name: OnAMPSubInvoiceSettled :exec +INSERT INTO invoice_events ( + added_at, event_type, invoice_id, set_id +) VALUES ( + $1, 5, $2, $3 +); diff --git a/sqldb/sqlc/queries/invoices.sql b/sqldb/sqlc/queries/invoices.sql index 335929e05..c36976a25 100644 --- a/sqldb/sqlc/queries/invoices.sql +++ b/sqldb/sqlc/queries/invoices.sql @@ -1,10 +1,10 @@ -- name: InsertInvoice :one INSERT INTO invoices ( hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, - payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, - created_at + payment_request, payment_request_hash, state, amount_paid_msat, is_amp, + is_hodl, is_keysend, created_at ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15 ) RETURNING id; -- name: InsertInvoiceFeature :exec @@ -19,35 +19,36 @@ SELECT * FROM invoice_features WHERE invoice_id = $1; --- name: DeleteInvoiceFeatures :exec -DELETE -FROM invoice_features -WHERE invoice_id = $1; - -- This method may return more than one invoice if filter using multiple fields -- from different invoices. It is the caller's responsibility to ensure that -- we bubble up an error in those cases. -- name: GetInvoice :many -SELECT * -FROM invoices +SELECT i.* +FROM invoices i +LEFT JOIN amp_sub_invoices a on i.id = a.invoice_id WHERE ( - id = sqlc.narg('add_index') OR + i.id = sqlc.narg('add_index') OR sqlc.narg('add_index') IS NULL ) AND ( - hash = sqlc.narg('hash') OR + i.hash = sqlc.narg('hash') OR sqlc.narg('hash') IS NULL ) AND ( - preimage = sqlc.narg('preimage') OR + i.preimage = sqlc.narg('preimage') OR sqlc.narg('preimage') IS NULL ) AND ( - payment_addr = sqlc.narg('payment_addr') OR + i.payment_addr = sqlc.narg('payment_addr') OR sqlc.narg('payment_addr') IS NULL +) AND ( + a.set_id = sqlc.narg('set_id') OR + sqlc.narg('set_id') IS NULL ) +GROUP BY i.id LIMIT 2; -- name: FilterInvoices :many -SELECT * +SELECT + invoices.* FROM invoices WHERE ( id >= sqlc.narg('add_index_get') OR @@ -55,6 +56,12 @@ WHERE ( ) AND ( id <= sqlc.narg('add_index_let') OR sqlc.narg('add_index_let') IS NULL +) AND ( + settle_index >= sqlc.narg('settle_index_get') OR + sqlc.narg('settle_index_get') IS NULL +) AND ( + settle_index <= sqlc.narg('settle_index_let') OR + sqlc.narg('settle_index_let') IS NULL ) AND ( state = sqlc.narg('state') OR sqlc.narg('state') IS NULL @@ -81,12 +88,25 @@ ORDER BY END DESC LIMIT @num_limit OFFSET @num_offset; --- name: UpdateInvoice :exec -UPDATE invoices -SET preimage=$2, state=$3, amount_paid_msat=$4 -WHERE id=$1; +-- name: UpdateInvoiceState :execresult +UPDATE invoices +SET state = $2, + preimage = COALESCE(preimage, $3), + settle_index = COALESCE(settle_index, $4), + settled_at = COALESCE(settled_at, $5) +WHERE id = $1; --- name: DeleteInvoice :exec +-- name: UpdateInvoiceAmountPaid :execresult +UPDATE invoices +SET amount_paid_msat = $2 +WHERE id = $1; + +-- name: NextInvoiceSettleIndex :one +UPDATE invoice_sequences SET current_value = current_value + 1 +WHERE name = 'settle_index' +RETURNING current_value; + +-- name: DeleteInvoice :execresult DELETE FROM invoices WHERE ( @@ -96,20 +116,25 @@ WHERE ( hash = sqlc.narg('hash') OR sqlc.narg('hash') IS NULL ) AND ( - preimage = sqlc.narg('preimage') OR - sqlc.narg('preimage') IS NULL + settle_index = sqlc.narg('settle_index') OR + sqlc.narg('settle_index') IS NULL ) AND ( payment_addr = sqlc.narg('payment_addr') OR sqlc.narg('payment_addr') IS NULL ); --- name: InsertInvoiceHTLC :exec +-- name: DeleteCanceledInvoices :execresult +DELETE +FROM invoices +WHERE state = 2; + +-- name: InsertInvoiceHTLC :one INSERT INTO invoice_htlcs ( htlc_id, chan_id, amount_msat, total_mpp_msat, accept_height, accept_time, expiry_height, state, resolve_time, invoice_id ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 -); +) RETURNING id; -- name: GetInvoiceHTLCs :many SELECT * @@ -118,25 +143,14 @@ WHERE invoice_id = $1; -- name: UpdateInvoiceHTLC :exec UPDATE invoice_htlcs -SET state=$2, resolve_time=$3 -WHERE id = $1; +SET state=$4, resolve_time=$5 +WHERE htlc_id = $1 AND chan_id = $2 AND invoice_id = $3; -- name: UpdateInvoiceHTLCs :exec UPDATE invoice_htlcs SET state=$2, resolve_time=$3 WHERE invoice_id = $1 AND resolve_time IS NULL; - --- name: DeleteInvoiceHTLC :exec -DELETE -FROM invoice_htlcs -WHERE htlc_id = $1; - --- name: DeleteInvoiceHTLCs :exec -DELETE -FROM invoice_htlcs -WHERE invoice_id = $1; - -- name: InsertInvoiceHTLCCustomRecord :exec INSERT INTO invoice_htlc_custom_records ( key, value, htlc_id @@ -148,49 +162,3 @@ INSERT INTO invoice_htlc_custom_records ( SELECT ihcr.htlc_id, key, value FROM invoice_htlcs ih JOIN invoice_htlc_custom_records ihcr ON ih.id=ihcr.htlc_id WHERE ih.invoice_id = $1; - --- name: DeleteInvoiceHTLCCustomRecords :exec -WITH htlc_ids AS ( - SELECT ih.id - FROM invoice_htlcs ih JOIN invoice_htlc_custom_records ihcr ON ih.id=ihcr.htlc_id - WHERE ih.invoice_id = $1 -) -DELETE -FROM invoice_htlc_custom_records -WHERE htlc_id IN (SELECT id FROM htlc_ids); - --- name: InsertInvoicePayment :one -INSERT INTO invoice_payments ( - invoice_id, amount_paid_msat, settled_at -) VALUES ( - $1, $2, $3 -) RETURNING id; - --- name: GetInvoicePayments :many -SELECT * -FROM invoice_payments -WHERE invoice_id = $1; - --- name: FilterInvoicePayments :many -SELECT - ip.id AS settle_index, ip.amount_paid_msat, ip.settled_at AS settle_date, - i.* -FROM invoice_payments ip JOIN invoices i ON ip.invoice_id = i.id -WHERE ( - ip.id >= sqlc.narg('settle_index_get') OR - sqlc.narg('settle_index_get') IS NULL -) AND ( - ip.settled_at >= sqlc.narg('settled_after') OR - sqlc.narg('settled_after') IS NULL -) -ORDER BY - CASE - WHEN sqlc.narg('reverse') = FALSE THEN ip.id - ELSE NULL - END ASC, - CASE - WHEN sqlc.narg('reverse') = TRUE THEN ip.id - ELSE NULL - END DESC -LIMIT @num_limit OFFSET @num_offset; - diff --git a/sqldb/sqlerrors.go b/sqldb/sqlerrors.go index c595bd528..a939ec499 100644 --- a/sqldb/sqlerrors.go +++ b/sqldb/sqlerrors.go @@ -1,3 +1,5 @@ +//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) + package sqldb import ( @@ -46,6 +48,11 @@ func parseSqliteError(sqliteErr *sqlite.Error) error { DBError: sqliteErr, } + case sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: + return &ErrSQLUniqueConstraintViolation{ + DBError: sqliteErr, + } + // Database is currently busy, so we'll need to try again. case sqlite3.SQLITE_BUSY: return &ErrSerializationError{ diff --git a/sqldb/sqlerrors_no_sqlite.go b/sqldb/sqlerrors_no_sqlite.go new file mode 100644 index 000000000..717d94152 --- /dev/null +++ b/sqldb/sqlerrors_no_sqlite.go @@ -0,0 +1,86 @@ +//go:build js || (windows && (arm || 386)) || (linux && (ppc64 || mips || mipsle || mips64)) + +package sqldb + +import ( + "errors" + "fmt" + + "github.com/jackc/pgconn" + "github.com/jackc/pgerrcode" +) + +var ( + // ErrRetriesExceeded is returned when a transaction is retried more + // than the max allowed valued without a success. + ErrRetriesExceeded = errors.New("db tx retries exceeded") +) + +// MapSQLError attempts to interpret a given error as a database agnostic SQL +// error. +func MapSQLError(err error) error { + // Attempt to interpret the error as a postgres error. + var pqErr *pgconn.PgError + if errors.As(err, &pqErr) { + return parsePostgresError(pqErr) + } + + // Return original error if it could not be classified as a database + // specific error. + return err +} + +// parsePostgresError attempts to parse a postgres error as a database agnostic +// SQL error. +func parsePostgresError(pqErr *pgconn.PgError) error { + switch pqErr.Code { + // Handle unique constraint violation error. + case pgerrcode.UniqueViolation: + return &ErrSQLUniqueConstraintViolation{ + DBError: pqErr, + } + + // Unable to serialize the transaction, so we'll need to try again. + case pgerrcode.SerializationFailure: + return &ErrSerializationError{ + DBError: pqErr, + } + + default: + return fmt.Errorf("unknown postgres error: %w", pqErr) + } +} + +// ErrSQLUniqueConstraintViolation is an error type which represents a database +// agnostic SQL unique constraint violation. +type ErrSQLUniqueConstraintViolation struct { + DBError error +} + +func (e ErrSQLUniqueConstraintViolation) Error() string { + return fmt.Sprintf("sql unique constraint violation: %v", e.DBError) +} + +// ErrSerializationError is an error type which represents a database agnostic +// error that a transaction couldn't be serialized with other concurrent db +// transactions. +type ErrSerializationError struct { + DBError error +} + +// Unwrap returns the wrapped error. +func (e ErrSerializationError) Unwrap() error { + return e.DBError +} + +// Error returns the error message. +func (e ErrSerializationError) Error() string { + return e.DBError.Error() +} + +// IsSerializationError returns true if the given error is a serialization +// error. +func IsSerializationError(err error) bool { + var serializationError *ErrSerializationError + return errors.As(err, &serializationError) +} diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index 8ecadd68d..705d5cc47 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -1,3 +1,5 @@ +//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) + package sqldb import ( @@ -6,7 +8,6 @@ import ( "net/url" "path/filepath" "testing" - "time" sqlite_migrate "github.com/golang-migrate/migrate/v4/database/sqlite" "github.com/lightningnetwork/lnd/sqldb/sqlc" @@ -23,32 +24,8 @@ const ( // sqliteTxLockImmediate is a dsn option used to ensure that write // transactions are started immediately. sqliteTxLockImmediate = "_txlock=immediate" - - // defaultMaxConns is the number of permitted active and idle - // connections. We want to limit this so it isn't unlimited. We use the - // same value for the number of idle connections as, this can speed up - // queries given a new connection doesn't need to be established each - // time. - defaultMaxConns = 25 - - // connIdleLifetime is the amount of time a connection can be idle. - connIdleLifetime = 5 * time.Minute ) -// SqliteConfig holds all the config arguments needed to interact with our -// sqlite DB. -// -//nolint:lll -type SqliteConfig struct { - // SkipMigrations if true, then all the tables will be created on start - // up if they don't already exist. - SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."` - - // DatabaseFileName is the full file path where the database file can be - // found. - DatabaseFileName string `long:"dbfile" description:"The full path to the database."` -} - // SqliteStore is a database store implementation that uses a sqlite backend. type SqliteStore struct { cfg *SqliteConfig @@ -58,7 +35,7 @@ type SqliteStore struct { // NewSqliteStore attempts to open a new sqlite database based on the passed // config. -func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) { +func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { // The set of pragma options are accepted using query options. For now // we only want to ensure that foreign key constraints are properly // enforced. @@ -107,7 +84,7 @@ func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) { // details on the formatting here, see the modernc.org/sqlite docs: // https://pkg.go.dev/modernc.org/sqlite#Driver.Open. dsn := fmt.Sprintf( - "%v?%v&%v", cfg.DatabaseFileName, sqliteOptions.Encode(), + "%v?%v&%v", dbPath, sqliteOptions.Encode(), sqliteTxLockImmediate, ) db, err := sql.Open("sqlite", dsn) @@ -133,8 +110,16 @@ func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) { return nil, err } + // We use INTEGER PRIMARY KEY for sqlite, because it acts as a + // ROWID alias which is 8 bytes big and also autoincrements. + // It's important to use the ROWID as a primary key because the + // key look ups are almost twice as fast + sqliteFS := newReplacerFS(sqlSchemas, map[string]string{ + "BIGINT PRIMARY KEY": "INTEGER PRIMARY KEY", + }) + err = applyMigrations( - sqlSchemas, driver, "sqlc/migrations", "sqlc", + sqliteFS, driver, "sqlc/migrations", "sqlc", ) if err != nil { return nil, err @@ -163,9 +148,8 @@ func NewTestSqliteDB(t *testing.T) *SqliteStore { // an in mem version to speed up tests dbFileName := filepath.Join(t.TempDir(), "tmp.db") sqlDB, err := NewSqliteStore(&SqliteConfig{ - DatabaseFileName: dbFileName, - SkipMigrations: false, - }) + SkipMigrations: false, + }, dbFileName) require.NoError(t, err) t.Cleanup(func() { diff --git a/sqldb/sqlutils.go b/sqldb/sqlutils.go new file mode 100644 index 000000000..b71c5f00c --- /dev/null +++ b/sqldb/sqlutils.go @@ -0,0 +1,54 @@ +package sqldb + +import ( + "database/sql" + "time" + + "golang.org/x/exp/constraints" +) + +// sqlInt32 turns a numerical integer type into the NullInt32 that sql/sqlc +// uses when an integer field can be permitted to be NULL. +// +// We use this constraints.Integer constraint here which maps to all signed and +// unsigned integer types. +func sqlInt32[T constraints.Integer](num T) sql.NullInt32 { + return sql.NullInt32{ + Int32: int32(num), + Valid: true, + } +} + +// sqlInt64 turns a numerical integer type into the NullInt64 that sql/sqlc +// uses when an integer field can be permitted to be NULL. +// +// We use this constraints.Integer constraint here which maps to all signed and +// unsigned integer types. +func sqlInt64[T constraints.Integer](num T) sql.NullInt64 { + return sql.NullInt64{ + Int64: int64(num), + Valid: true, + } +} + +// sqlStr turns a string into the NullString that sql/sqlc uses when a string +// can be permitted to be NULL. +func sqlStr(s string) sql.NullString { + if s == "" { + return sql.NullString{} + } + + return sql.NullString{ + String: s, + Valid: true, + } +} + +// sqlTime turns a time.Time into the NullTime that sql/sqlc uses when a time +// can be permitted to be NULL. +func sqlTime(t time.Time) sql.NullTime { + return sql.NullTime{ + Time: t, + Valid: true, + } +}