Merge pull request #8052 from bhandras/sql-invoices

sqldb: `InvoiceDB` implementation

.github/workflows/main.yml

@@ -252,6 +252,10 @@ jobs:
             args: backend=bitcoind dbbackend=postgres
           - name: bitcoind-sqlite
             args: backend=bitcoind dbbackend=sqlite
+          - name: bitcoind-postgres-nativesql
+            args: backend=bitcoind dbbackend=postgres nativesql=true
+          - name: bitcoind-sqlite-nativesql
+            args: backend=bitcoind dbbackend=sqlite nativesql=true
           - name: neutrino
             args: backend=neutrino
     steps:

config_builder.go

@@ -3,6 +3,7 @@ package lnd
 import (
 	"bytes"
 	"context"
+	"database/sql"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -31,6 +32,7 @@ import (
 	"github.com/lightningnetwork/lnd/chainntnfs"
 	"github.com/lightningnetwork/lnd/chainreg"
 	"github.com/lightningnetwork/lnd/channeldb"
+	"github.com/lightningnetwork/lnd/clock"
 	"github.com/lightningnetwork/lnd/invoices"
 	"github.com/lightningnetwork/lnd/keychain"
 	"github.com/lightningnetwork/lnd/kvdb"
@@ -42,6 +44,7 @@ import (
 	"github.com/lightningnetwork/lnd/macaroons"
 	"github.com/lightningnetwork/lnd/rpcperms"
 	"github.com/lightningnetwork/lnd/signal"
+	"github.com/lightningnetwork/lnd/sqldb"
 	"github.com/lightningnetwork/lnd/walletunlocker"
 	"github.com/lightningnetwork/lnd/watchtower"
 	"github.com/lightningnetwork/lnd/watchtower/wtclient"
@@ -869,6 +872,11 @@ type DatabaseInstances struct {
 	// WalletDB is the configuration for loading the wallet database using
 	// the btcwallet's loader.
 	WalletDB btcwallet.LoaderOption
+
+	// NativeSQLStore is a pointer to a native SQL store that can be used
+	// for native SQL queries for tables that already support it. This may
+	// be nil if the use-native-sql flag was not set.
+	NativeSQLStore *sqldb.BaseDB
 }
 
 // DefaultDatabaseBuilder is a type that builds the default database backends
@@ -923,10 +931,11 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 	// state DB point to the same local or remote DB and the same namespace
 	// within that DB.
 	dbs := &DatabaseInstances{
-		HeightHintDB: databaseBackends.HeightHintDB,
+		HeightHintDB:   databaseBackends.HeightHintDB,
-		MacaroonDB:   databaseBackends.MacaroonDB,
+		MacaroonDB:     databaseBackends.MacaroonDB,
-		DecayedLogDB: databaseBackends.DecayedLogDB,
+		DecayedLogDB:   databaseBackends.DecayedLogDB,
-		WalletDB:     databaseBackends.WalletDB,
+		WalletDB:       databaseBackends.WalletDB,
+		NativeSQLStore: databaseBackends.NativeSQLStore,
 	}
 	cleanUp := func() {
 		// We can just close the returned close functions directly. Even
@@ -1011,11 +1020,21 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 	// using the same struct (and DB backend) instance.
 	dbs.ChanStateDB = dbs.GraphDB
 
-	// For now the only InvoiceDB implementation is the *channeldb.DB.
+	// Instantiate a native SQL invoice store if the flag is set.
-	//
+	if d.cfg.DB.UseNativeSQL {
-	// TODO(positiveblue): use a sql first implementation for this
+		executor := sqldb.NewTransactionExecutor(
-	// interface.
+			dbs.NativeSQLStore,
-	dbs.InvoiceDB = dbs.GraphDB
+			func(tx *sql.Tx) sqldb.InvoiceQueries {
+				return dbs.NativeSQLStore.WithTx(tx)
+			},
+		)
+
+		dbs.InvoiceDB = sqldb.NewInvoiceStore(
+			executor, clock.NewDefaultClock(),
+		)
+	} else {
+		dbs.InvoiceDB = dbs.GraphDB
+	}
 
 	// Wrap the watchtower client DB and make sure we clean up.
 	if cfg.WtClient.Active {
@@ -1298,8 +1317,9 @@ func initNeutrinoBackend(ctx context.Context, cfg *Config, chainDir string,
 	)
 	switch {
 	case cfg.DB.Backend == kvdb.SqliteBackendName:
+		sqliteConfig := lncfg.GetSqliteConfigKVDB(cfg.DB.Sqlite)
 		db, err = kvdb.Open(
-			kvdb.SqliteBackendName, ctx, cfg.DB.Sqlite, dbPath,
+			kvdb.SqliteBackendName, ctx, sqliteConfig, dbPath,
 			lncfg.SqliteNeutrinoDBName, lncfg.NSNeutrinoDB,
 		)
 

docs/release-notes/release-notes-0.18.0.md

@@ -339,6 +339,10 @@
 * [Refactor UpdateInvoice](https://github.com/lightningnetwork/lnd/pull/8100) to
   make it simpler to adjust code to also support SQL InvoiceDB implementation.
 
+* [InvoiceDB implementation](https://github.com/lightningnetwork/lnd/pull/8052)
+  for SQL backends enabling new users to optionally use  an experimental native
+  SQL invoices database.
+
 ## Code Health
 
 * [Remove database pointers](https://github.com/lightningnetwork/lnd/pull/8117) 

invoices/invoiceregistry_test.go

@@ -3,6 +3,7 @@ package invoices_test
 import (
 	"context"
 	"crypto/rand"
+	"database/sql"
 	"fmt"
 	"math"
 	"testing"
@@ -18,6 +19,7 @@ import (
 	"github.com/lightningnetwork/lnd/lntypes"
 	"github.com/lightningnetwork/lnd/lnwire"
 	"github.com/lightningnetwork/lnd/record"
+	"github.com/lightningnetwork/lnd/sqldb"
 	"github.com/stretchr/testify/require"
 )
 
@@ -114,12 +116,60 @@ func TestInvoiceRegistry(t *testing.T) {
 		return db, testClock
 	}
 
+	// First create a shared Postgres instance so we don't spawn a new
+	// docker container for each test.
+	pgFixture := sqldb.NewTestPgFixture(
+		t, sqldb.DefaultPostgresFixtureLifetime,
+	)
+	t.Cleanup(func() {
+		pgFixture.TearDown(t)
+	})
+
+	makeSQLDB := func(t *testing.T, sqlite bool) (invpkg.InvoiceDB,
+		*clock.TestClock) {
+
+		var db *sqldb.BaseDB
+		if sqlite {
+			db = sqldb.NewTestSqliteDB(t).BaseDB
+		} else {
+			db = sqldb.NewTestPostgresDB(t, pgFixture).BaseDB
+		}
+
+		executor := sqldb.NewTransactionExecutor(
+			db, func(tx *sql.Tx) sqldb.InvoiceQueries {
+				return db.WithTx(tx)
+			},
+		)
+
+		testClock := clock.NewTestClock(testNow)
+
+		return sqldb.NewInvoiceStore(executor, testClock), testClock
+	}
+
 	for _, test := range testList {
 		test := test
 
-		t.Run(test.name, func(t *testing.T) {
+		t.Run(test.name+"_KV", func(t *testing.T) {
 			test.test(t, makeKeyValueDB)
 		})
+
+		t.Run(test.name+"_SQLite", func(t *testing.T) {
+			test.test(t,
+				func(t *testing.T) (
+					invpkg.InvoiceDB, *clock.TestClock) {
+
+					return makeSQLDB(t, true)
+				})
+		})
+
+		t.Run(test.name+"_Postgres", func(t *testing.T) {
+			test.test(t,
+				func(t *testing.T) (
+					invpkg.InvoiceDB, *clock.TestClock) {
+
+					return makeSQLDB(t, false)
+				})
+		})
 	}
 }
 

invoices/invoices_test.go

@@ -3,8 +3,10 @@ package invoices_test
 import (
 	"context"
 	"crypto/rand"
+	"database/sql"
 	"fmt"
 	"math"
+	"strings"
 	"testing"
 	"time"
 
@@ -16,6 +18,7 @@ import (
 	"github.com/lightningnetwork/lnd/lntypes"
 	"github.com/lightningnetwork/lnd/lnwire"
 	"github.com/lightningnetwork/lnd/record"
+	"github.com/lightningnetwork/lnd/sqldb"
 	"github.com/stretchr/testify/require"
 )
 
@@ -34,6 +37,23 @@ var (
 	testNow = time.Unix(1, 0)
 )
 
+// randBytesToString will return a "safe" string from a byte slice. This is
+// used to generate random strings for the invoice payment request.
+func randBytesToString(buf []byte) (string, error) {
+	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
+	var stringBuilder strings.Builder
+
+	stringBuilder.Grow(len(buf))
+	for i := 0; i < len(buf); i++ {
+		ch := charset[int(buf[i])%len(charset)]
+		if err := stringBuilder.WriteByte(ch); err != nil {
+			return "", err
+		}
+	}
+
+	return stringBuilder.String(), nil
+}
+
 func randInvoice(value lnwire.MilliSatoshi) (*invpkg.Invoice, error) {
 	var (
 		pre     lntypes.Preimage
@@ -68,7 +88,11 @@ func randInvoice(value lnwire.MilliSatoshi) (*invpkg.Invoice, error) {
 		return nil, err
 	}
 	if r[0]&1 == 0 {
-		i.PaymentRequest = r[:]
+		paymentReq, err := randBytesToString(r[:])
+		if err != nil {
+			return nil, err
+		}
+		i.PaymentRequest = []byte(paymentReq)
 	} else {
 		i.PaymentRequest = []byte("")
 	}
@@ -77,11 +101,13 @@ func randInvoice(value lnwire.MilliSatoshi) (*invpkg.Invoice, error) {
 }
 
 // settleTestInvoice settles a test invoice.
-func settleTestInvoice(invoice *invpkg.Invoice, settleIndex uint64) {
+func settleTestInvoice(invoice *invpkg.Invoice, htlcID uint64,
+	settleIndex uint64) {
+
 	invoice.SettleDate = testNow
 	invoice.AmtPaid = invoice.Terms.Value
 	invoice.State = invpkg.ContractSettled
-	invoice.Htlcs[models.CircuitKey{}] = &invpkg.InvoiceHTLC{
+	invoice.Htlcs[models.CircuitKey{HtlcID: htlcID}] = &invpkg.InvoiceHTLC{
 		Amt:           invoice.Terms.Value,
 		AcceptTime:    testNow,
 		ResolveTime:   testNow,
@@ -196,12 +222,53 @@ func TestInvoices(t *testing.T) {
 		return db
 	}
 
+	// First create a shared Postgres instance so we don't spawn a new
+	// docker container for each test.
+	pgFixture := sqldb.NewTestPgFixture(
+		t, sqldb.DefaultPostgresFixtureLifetime,
+	)
+	t.Cleanup(func() {
+		pgFixture.TearDown(t)
+	})
+
+	makeSQLDB := func(t *testing.T, sqlite bool) invpkg.InvoiceDB {
+		var db *sqldb.BaseDB
+		if sqlite {
+			db = sqldb.NewTestSqliteDB(t).BaseDB
+		} else {
+			db = sqldb.NewTestPostgresDB(t, pgFixture).BaseDB
+		}
+
+		executor := sqldb.NewTransactionExecutor(
+			db, func(tx *sql.Tx) sqldb.InvoiceQueries {
+				return db.WithTx(tx)
+			},
+		)
+
+		testClock := clock.NewTestClock(testNow)
+
+		return sqldb.NewInvoiceStore(executor, testClock)
+	}
+
 	for _, test := range testList {
 		test := test
-
+		t.Run(test.name+"_KV", func(t *testing.T) {
-		t.Run(test.name, func(t *testing.T) {
 			test.test(t, makeKeyValueDB)
 		})
+
+		t.Run(test.name+"_SQLite", func(t *testing.T) {
+			test.test(t,
+				func(t *testing.T) invpkg.InvoiceDB {
+					return makeSQLDB(t, true)
+				})
+		})
+
+		t.Run(test.name+"_Postgres", func(t *testing.T) {
+			test.test(t,
+				func(t *testing.T) invpkg.InvoiceDB {
+					return makeSQLDB(t, false)
+				})
+		})
 	}
 }
 
@@ -333,7 +400,7 @@ func testInvoiceWorkflowImpl(t *testing.T, test invWorkflowTest,
 	// now have the settled bit toggle to true and a non-default
 	// SettledDate
 	payAmt := fakeInvoice.Terms.Value * 2
-	_, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(payAmt))
+	_, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(0, payAmt))
 	require.NoError(t, err, "unable to settle invoice")
 	dbInvoice2, err := db.LookupInvoice(ctxb, ref)
 	require.NoError(t, err, "unable to fetch invoice")
@@ -512,7 +579,7 @@ func testFailInvoiceLookupMPPPayAddrOnly(t *testing.T,
 	// for the given HTLC.
 	ref := invpkg.InvoiceRefByHashAndAddr(payHash, payAddr)
 	_, err = db.LookupInvoice(ctxb, ref)
-	require.Equal(t, invpkg.ErrInvoiceNotFound, err)
+	require.ErrorIs(t, err, invpkg.ErrInvoiceNotFound)
 }
 
 // testInvRefEquivocation asserts that retrieving or updating an invoice using
@@ -690,7 +757,7 @@ func testInvoiceCancelSingleHtlcAMP(t *testing.T,
 
 	// At this point, we should detect that 3k satoshis total has been
 	// paid.
-	require.Equal(t, dbInvoice.AmtPaid, amt*3)
+	require.EqualValues(t, amt*3, dbInvoice.AmtPaid)
 
 	callback := func(
 		invoice *invpkg.Invoice) (*invpkg.InvoiceUpdateDesc, error) {
@@ -902,8 +969,11 @@ func testInvoiceAddTimeSeries(t *testing.T,
 	_, err = db.InvoicesSettledSince(ctxb, 0)
 	require.NoError(t, err)
 
-	var settledInvoices []invpkg.Invoice
+	var (
-	var settleIndex uint64 = 1
+		settledInvoices []invpkg.Invoice
+		settleIndex     uint64 = 1
+		htlcID          uint64 = 0
+	)
 	// We'll now only settle the latter half of each of those invoices.
 	for i := 10; i < len(invoices); i++ {
 		invoice := &invoices[i]
@@ -912,15 +982,18 @@ func testInvoiceAddTimeSeries(t *testing.T,
 
 		ref := invpkg.InvoiceRefByHash(paymentHash)
 		_, err := db.UpdateInvoice(
-			ctxb, ref, nil, getUpdateInvoice(invoice.Terms.Value),
+			ctxb, ref, nil, getUpdateInvoice(
+				htlcID, invoice.Terms.Value,
+			),
 		)
 		if err != nil {
 			t.Fatalf("unable to settle invoice: %v", err)
 		}
 
 		// Create the settled invoice for the expectation set.
-		settleTestInvoice(invoice, settleIndex)
+		settleTestInvoice(invoice, htlcID, settleIndex)
 		settleIndex++
+		htlcID++
 
 		settledInvoices = append(settledInvoices, *invoice)
 	}
@@ -960,7 +1033,10 @@ func testInvoiceAddTimeSeries(t *testing.T,
 			t.Fatalf("unable to query: %v", err)
 		}
 
-		require.Equal(t, len(query.resp), len(resp))
+		require.Equal(
+			t, len(query.resp), len(resp),
+			fmt.Sprintf("test: #%v", i),
+		)
 
 		for j := 0; j < len(query.resp); j++ {
 			require.Equal(t,
@@ -1138,7 +1214,11 @@ func testSettleIndexAmpPayments(t *testing.T,
 			AddIndex: testInvoice.AddIndex,
 		},
 	})
-	require.Nil(t, err)
+	require.NoError(t, err)
+
+	settledInvoices, err = db.InvoicesSettledSince(ctxb, 0)
+	require.NoError(t, err)
+	require.Len(t, settledInvoices, 0)
 }
 
 // testFetchPendingInvoices tests that we can fetch all pending invoices from
@@ -1158,7 +1238,11 @@ func testFetchPendingInvoices(t *testing.T,
 	require.Empty(t, pending)
 
 	const numInvoices = 20
-	var settleIndex uint64 = 1
+	var (
+		settleIndex uint64 = 1
+		htlcID      uint64 = 0
+	)
+
 	pendingInvoices := make(map[lntypes.Hash]invpkg.Invoice)
 
 	for i := 1; i <= numInvoices; i++ {
@@ -1182,11 +1266,14 @@ func testFetchPendingInvoices(t *testing.T,
 		}
 
 		ref := invpkg.InvoiceRefByHash(paymentHash)
-		_, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(amt))
+		_, err = db.UpdateInvoice(
+			ctxb, ref, nil, getUpdateInvoice(htlcID, amt),
+		)
 		require.NoError(t, err)
 
-		settleTestInvoice(invoice, settleIndex)
+		settleTestInvoice(invoice, htlcID, settleIndex)
 		settleIndex++
+		htlcID++
 	}
 
 	// Fetch all pending invoices.
@@ -1219,7 +1306,7 @@ func testDuplicateSettleInvoice(t *testing.T,
 	// With the invoice in the DB, we'll now attempt to settle the invoice.
 	ref := invpkg.InvoiceRefByHash(payHash)
 	dbInvoice, err := db.UpdateInvoice(
-		ctxb, ref, nil, getUpdateInvoice(amt),
+		ctxb, ref, nil, getUpdateInvoice(0, amt),
 	)
 	require.NoError(t, err, "unable to settle invoice")
 
@@ -1244,7 +1331,9 @@ func testDuplicateSettleInvoice(t *testing.T,
 
 	// If we try to settle the invoice again, then we should get the very
 	// same invoice back, but with an error this time.
-	dbInvoice, err = db.UpdateInvoice(ctxb, ref, nil, getUpdateInvoice(amt))
+	dbInvoice, err = db.UpdateInvoice(
+		ctxb, ref, nil, getUpdateInvoice(1, amt),
+	)
 	require.ErrorIs(t, err, invpkg.ErrInvoiceAlreadySettled)
 
 	if dbInvoice == nil {
@@ -1269,9 +1358,12 @@ func testQueryInvoices(t *testing.T,
 	// assume that the index of the invoice within the database is the same
 	// as the amount of the invoice itself.
 	const numInvoices = 50
-	var settleIndex uint64 = 1
+	var (
-	var invoices []invpkg.Invoice
+		settleIndex     uint64 = 1
-	var pendingInvoices []invpkg.Invoice
+		htlcID          uint64 = 0
+		invoices        []invpkg.Invoice
+		pendingInvoices []invpkg.Invoice
+	)
 
 	ctxb := context.Background()
 	for i := 1; i <= numInvoices; i++ {
@@ -1296,15 +1388,16 @@ func testQueryInvoices(t *testing.T,
 		if i%2 == 0 {
 			ref := invpkg.InvoiceRefByHash(paymentHash)
 			_, err := db.UpdateInvoice(
-				ctxb, ref, nil, getUpdateInvoice(amt),
+				ctxb, ref, nil, getUpdateInvoice(htlcID, amt),
 			)
 			if err != nil {
 				t.Fatalf("unable to settle invoice: %v", err)
 			}
 
 			// Create the settled invoice for the expectation set.
-			settleTestInvoice(invoice, settleIndex)
+			settleTestInvoice(invoice, htlcID, settleIndex)
 			settleIndex++
+			htlcID++
 		} else {
 			pendingInvoices = append(pendingInvoices, *invoice)
 		}
@@ -1649,7 +1742,8 @@ func testQueryInvoices(t *testing.T,
 			t.Fatalf("unable to query invoice database: %v", err)
 		}
 
-		require.Equal(t, len(testCase.expected), len(response.Invoices))
+		require.Equal(t, len(testCase.expected), len(response.Invoices),
+			fmt.Sprintf("test: #%v", i))
 
 		for j, expected := range testCase.expected {
 			require.Equal(t,
@@ -1662,7 +1756,9 @@ func testQueryInvoices(t *testing.T,
 
 // getUpdateInvoice returns an invoice update callback that, when called,
 // settles the invoice with the given amount.
-func getUpdateInvoice(amt lnwire.MilliSatoshi) invpkg.InvoiceUpdateCallback {
+func getUpdateInvoice(htlcID uint64,
+	amt lnwire.MilliSatoshi) invpkg.InvoiceUpdateCallback {
+
 	return func(invoice *invpkg.Invoice) (*invpkg.InvoiceUpdateDesc,
 		error) {
 
@@ -1672,7 +1768,7 @@ func getUpdateInvoice(amt lnwire.MilliSatoshi) invpkg.InvoiceUpdateCallback {
 
 		noRecords := make(record.CustomSet)
 		htlcs := map[models.CircuitKey]*invpkg.HtlcAcceptDesc{
-			{}: {
+			{HtlcID: htlcID}: {
 				Amt:           amt,
 				CustomRecords: noRecords,
 			},
@@ -2082,7 +2178,10 @@ func testSetIDIndex(t *testing.T, makeDB func(t *testing.T) invpkg.InvoiceDB) {
 	)
 	_, err = db.UpdateInvoice(
 		ctxb, ref2, (*invpkg.SetID)(setID),
-		updateAcceptAMPHtlc(0, amt, setID, true),
+		// Note: we need a unique HTLC ID here otherwise the update will
+		// be rejected as a duplicate (due to SQL unique constraint
+		// violation).
+		updateAcceptAMPHtlc(55, amt, setID, true),
 	)
 	require.Equal(t, invpkg.ErrDuplicateSetID{SetID: *setID}, err)
 
@@ -2366,7 +2465,7 @@ func testUnexpectedInvoicePreimage(t *testing.T,
 	// payment hash against which to validate the preimage.
 	_, err = db.UpdateInvoice(
 		ctxb, invpkg.InvoiceRefByAddr(invoice.Terms.PaymentAddr), nil,
-		getUpdateInvoice(invoice.Terms.Value),
+		getUpdateInvoice(0, invoice.Terms.Value),
 	)
 
 	// Assert that we get ErrUnexpectedInvoicePreimage.
@@ -2501,7 +2600,9 @@ func testDeleteInvoices(t *testing.T,
 		if i == 1 {
 			invoice, err = db.UpdateInvoice(
 				ctxb, invpkg.InvoiceRefByHash(paymentHash), nil,
-				getUpdateInvoice(invoice.Terms.Value),
+				getUpdateInvoice(
+					uint64(i), invoice.Terms.Value,
+				),
 			)
 			require.NoError(t, err, "unable to settle invoice")
 		}

invoices/test_utils_test.go

@@ -231,7 +231,7 @@ func timeout() func() {
 
 	go func() {
 		select {
-		case <-time.After(5 * time.Second):
+		case <-time.After(10 * time.Second):
 			err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
 			if err != nil {
 				panic(fmt.Sprintf("error writing to std out "+

itest/lnd_test.go

@@ -71,6 +71,9 @@ var (
 	dbBackendFlag = flag.String("dbbackend", "bbolt", "Database backend "+
 		"(bbolt, etcd, postgres)")
 
+	nativeSQLFlag = flag.Bool("nativesql", false, "Database backend to "+
+		"use native SQL when applicable (only for sqlite and postgres")
+
 	// lndExecutable is the full path to the lnd binary.
 	lndExecutable = flag.String(
 		"lndexec", itestLndBinary, "full path to lnd binary",
@@ -95,7 +98,7 @@ func TestLightningNetworkDaemon(t *testing.T) {
 	// Get the binary path and setup the harness test.
 	binary := getLndBinary(t)
 	harnessTest := lntest.SetupHarness(
-		t, binary, *dbBackendFlag, feeService,
+		t, binary, *dbBackendFlag, *nativeSQLFlag, feeService,
 	)
 	defer harnessTest.Stop()
 

lncfg/db.go

@@ -3,6 +3,7 @@ package lncfg
 import (
 	"context"
 	"fmt"
+	"path"
 	"path/filepath"
 	"time"
 
@@ -14,6 +15,7 @@ import (
 	"github.com/lightningnetwork/lnd/kvdb/sqlite"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/lnwallet/btcwallet"
+	"github.com/lightningnetwork/lnd/sqldb"
 )
 
 const (
@@ -28,6 +30,7 @@ const (
 	SqliteChainDBName    = "chain.sqlite"
 	SqliteNeutrinoDBName = "neutrino.sqlite"
 	SqliteTowerDBName    = "watchtower.sqlite"
+	SqliteNativeDBName   = "lnd.sqlite"
 
 	BoltBackend                = "bolt"
 	EtcdBackend                = "etcd"
@@ -78,9 +81,11 @@ type DB struct {
 
 	Bolt *kvdb.BoltConfig `group:"bolt" namespace:"bolt" description:"Bolt settings."`
 
-	Postgres *postgres.Config `group:"postgres" namespace:"postgres" description:"Postgres settings."`
+	Postgres *sqldb.PostgresConfig `group:"postgres" namespace:"postgres" description:"Postgres settings."`
 
-	Sqlite *sqlite.Config `group:"sqlite" namespace:"sqlite" description:"Sqlite settings."`
+	Sqlite *sqldb.SqliteConfig `group:"sqlite" namespace:"sqlite" description:"Sqlite settings."`
+
+	UseNativeSQL bool `long:"use-native-sql" description:"Use native SQL for tables that already support it."`
 
 	NoGraphCache bool `long:"no-graph-cache" description:"Don't use the in-memory graph cache for path finding. Much slower but uses less RAM. Can only be used with a bolt database backend."`
 
@@ -103,13 +108,14 @@ func DefaultDB() *DB {
 			// Allow at most 32 MiB messages by default.
 			MaxMsgSize: 32768 * 1024,
 		},
-		Postgres: &postgres.Config{
+		Postgres: &sqldb.PostgresConfig{
 			MaxConnections: defaultPostgresMaxConnections,
 		},
-		Sqlite: &sqlite.Config{
+		Sqlite: &sqldb.SqliteConfig{
 			MaxConnections: defaultSqliteMaxConnections,
 			BusyTimeout:    defaultSqliteBusyTimeout,
 		},
+		UseNativeSQL: false,
 	}
 }
 
@@ -118,8 +124,8 @@ func (db *DB) Validate() error {
 	switch db.Backend {
 	case BoltBackend, SqliteBackend:
 	case PostgresBackend:
-		if db.Postgres.Dsn == "" {
+		if err := db.Postgres.Validate(); err != nil {
-			return fmt.Errorf("postgres dsn must be set")
+			return err
 		}
 
 	case EtcdBackend:
@@ -214,6 +220,11 @@ type DatabaseBackends struct {
 	// the underlying wallet database from.
 	WalletDB btcwallet.LoaderOption
 
+	// NativeSQLStore is a pointer to a native SQL store that can be used
+	// for native SQL queries for tables that already support it. This may
+	// be nil if the use-native-sql flag was not set.
+	NativeSQLStore *sqldb.BaseDB
+
 	// Remote indicates whether the database backends are remote, possibly
 	// replicated instances or local bbolt or sqlite backed databases.
 	Remote bool
@@ -223,6 +234,26 @@ type DatabaseBackends struct {
 	CloseFuncs map[string]func() error
 }
 
+// GetPostgresConfigKVDB converts a sqldb.PostgresConfig to a kvdb
+// postgres.Config.
+func GetPostgresConfigKVDB(cfg *sqldb.PostgresConfig) *postgres.Config {
+	return &postgres.Config{
+		Dsn:            cfg.Dsn,
+		Timeout:        cfg.Timeout,
+		MaxConnections: cfg.MaxConnections,
+	}
+}
+
+// GetSqliteConfigKVDB converts a sqldb.SqliteConfig to a kvdb sqlite.Config.
+func GetSqliteConfigKVDB(cfg *sqldb.SqliteConfig) *sqlite.Config {
+	return &sqlite.Config{
+		Timeout:        cfg.Timeout,
+		BusyTimeout:    cfg.BusyTimeout,
+		MaxConnections: cfg.MaxConnections,
+		PragmaOptions:  cfg.PragmaOptions,
+	}
+}
+
 // GetBackends returns a set of kvdb.Backends as set in the DB config.
 func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 	walletDBPath, towerServerDBPath string, towerClientEnabled,
@@ -342,9 +373,14 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		}, nil
 
 	case PostgresBackend:
+		// Convert the sqldb PostgresConfig to a kvdb postgres.Config.
+		// This is a temporary measure until we migrate all kvdb SQL
+		// users to native SQL.
+		postgresConfig := GetPostgresConfigKVDB(db.Postgres)
+
 		postgresBackend, err := kvdb.Open(
 			kvdb.PostgresBackendName, ctx,
-			db.Postgres, NSChannelDB,
+			postgresConfig, NSChannelDB,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("error opening postgres graph "+
@@ -354,7 +390,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 
 		postgresMacaroonBackend, err := kvdb.Open(
 			kvdb.PostgresBackendName, ctx,
-			db.Postgres, NSMacaroonDB,
+			postgresConfig, NSMacaroonDB,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("error opening postgres "+
@@ -364,7 +400,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 
 		postgresDecayedLogBackend, err := kvdb.Open(
 			kvdb.PostgresBackendName, ctx,
-			db.Postgres, NSDecayedLogDB,
+			postgresConfig, NSDecayedLogDB,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("error opening postgres "+
@@ -374,7 +410,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 
 		postgresTowerClientBackend, err := kvdb.Open(
 			kvdb.PostgresBackendName, ctx,
-			db.Postgres, NSTowerClientDB,
+			postgresConfig, NSTowerClientDB,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("error opening postgres tower "+
@@ -384,7 +420,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 
 		postgresTowerServerBackend, err := kvdb.Open(
 			kvdb.PostgresBackendName, ctx,
-			db.Postgres, NSTowerServerDB,
+			postgresConfig, NSTowerServerDB,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("error opening postgres tower "+
@@ -394,7 +430,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 
 		postgresWalletBackend, err := kvdb.Open(
 			kvdb.PostgresBackendName, ctx,
-			db.Postgres, NSWalletDB,
+			postgresConfig, NSWalletDB,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("error opening postgres macaroon "+
@@ -402,6 +438,20 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		}
 		closeFuncs[NSWalletDB] = postgresWalletBackend.Close
 
+		var nativeSQLStore *sqldb.BaseDB
+		if db.UseNativeSQL {
+			nativePostgresStore, err := sqldb.NewPostgresStore(
+				db.Postgres,
+			)
+			if err != nil {
+				return nil, fmt.Errorf("error opening "+
+					"native postgres store: %v", err)
+			}
+
+			nativeSQLStore = nativePostgresStore.BaseDB
+			closeFuncs[PostgresBackend] = nativePostgresStore.Close
+		}
+
 		// Warn if the user is trying to switch over to a Postgres DB
 		// while there is a wallet or channel bbolt DB still present.
 		warnExistingBoltDBs(
@@ -429,11 +479,17 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 			WalletDB: btcwallet.LoaderWithExternalWalletDB(
 				postgresWalletBackend,
 			),
-			Remote:     true,
+			NativeSQLStore: nativeSQLStore,
-			CloseFuncs: closeFuncs,
+			Remote:         true,
+			CloseFuncs:     closeFuncs,
 		}, nil
 
 	case SqliteBackend:
+		// Convert the sqldb SqliteConfig to a kvdb sqlite.Config.
+		// This is a temporary measure until we migrate all kvdb SQL
+		// users to native SQL.
+		sqliteConfig := GetSqliteConfigKVDB(db.Sqlite)
+
 		// Note that for sqlite, we put kv tables for the channel.db,
 		// wtclient.db and sphinxreplay.db all in the channel.sqlite db.
 		// The tables for wallet.db and macaroon.db are in the
@@ -445,7 +501,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		// would cause deadlocks in the code due to the wallet db often
 		// being accessed during a write to another db.
 		sqliteBackend, err := kvdb.Open(
-			kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath,
+			kvdb.SqliteBackendName, ctx, sqliteConfig, chanDBPath,
 			SqliteChannelDBName, NSChannelDB,
 		)
 		if err != nil {
@@ -455,7 +511,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		closeFuncs[NSChannelDB] = sqliteBackend.Close
 
 		sqliteMacaroonBackend, err := kvdb.Open(
-			kvdb.SqliteBackendName, ctx, db.Sqlite, walletDBPath,
+			kvdb.SqliteBackendName, ctx, sqliteConfig, walletDBPath,
 			SqliteChainDBName, NSMacaroonDB,
 		)
 		if err != nil {
@@ -465,7 +521,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		closeFuncs[NSMacaroonDB] = sqliteMacaroonBackend.Close
 
 		sqliteDecayedLogBackend, err := kvdb.Open(
-			kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath,
+			kvdb.SqliteBackendName, ctx, sqliteConfig, chanDBPath,
 			SqliteChannelDBName, NSDecayedLogDB,
 		)
 		if err != nil {
@@ -475,7 +531,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		closeFuncs[NSDecayedLogDB] = sqliteDecayedLogBackend.Close
 
 		sqliteTowerClientBackend, err := kvdb.Open(
-			kvdb.SqliteBackendName, ctx, db.Sqlite, chanDBPath,
+			kvdb.SqliteBackendName, ctx, sqliteConfig, chanDBPath,
 			SqliteChannelDBName, NSTowerClientDB,
 		)
 		if err != nil {
@@ -485,7 +541,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		closeFuncs[NSTowerClientDB] = sqliteTowerClientBackend.Close
 
 		sqliteTowerServerBackend, err := kvdb.Open(
-			kvdb.SqliteBackendName, ctx, db.Sqlite,
+			kvdb.SqliteBackendName, ctx, sqliteConfig,
 			towerServerDBPath, SqliteTowerDBName, NSTowerServerDB,
 		)
 		if err != nil {
@@ -495,7 +551,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		closeFuncs[NSTowerServerDB] = sqliteTowerServerBackend.Close
 
 		sqliteWalletBackend, err := kvdb.Open(
-			kvdb.SqliteBackendName, ctx, db.Sqlite, walletDBPath,
+			kvdb.SqliteBackendName, ctx, sqliteConfig, walletDBPath,
 			SqliteChainDBName, NSWalletDB,
 		)
 		if err != nil {
@@ -504,6 +560,21 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 		}
 		closeFuncs[NSWalletDB] = sqliteWalletBackend.Close
 
+		var nativeSQLStore *sqldb.BaseDB
+		if db.UseNativeSQL {
+			nativeSQLiteStore, err := sqldb.NewSqliteStore(
+				db.Sqlite,
+				path.Join(chanDBPath, SqliteNativeDBName),
+			)
+			if err != nil {
+				return nil, fmt.Errorf("error opening "+
+					"native SQLite store: %v", err)
+			}
+
+			nativeSQLStore = nativeSQLiteStore.BaseDB
+			closeFuncs[SqliteBackend] = nativeSQLiteStore.Close
+		}
+
 		// Warn if the user is trying to switch over to a sqlite DB
 		// while there is a wallet or channel bbolt DB still present.
 		warnExistingBoltDBs(
@@ -531,7 +602,8 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath,
 			WalletDB: btcwallet.LoaderWithExternalWalletDB(
 				sqliteWalletBackend,
 			),
-			CloseFuncs: closeFuncs,
+			NativeSQLStore: nativeSQLStore,
+			CloseFuncs:     closeFuncs,
 		}, nil
 	}
 

lntest/harness.go

@@ -103,14 +103,14 @@ type HarnessTest struct {
 // NewHarnessTest creates a new instance of a harnessTest from a regular
 // testing.T instance.
 func NewHarnessTest(t *testing.T, lndBinary string, feeService WebFeeService,
-	dbBackend node.DatabaseBackend) *HarnessTest {
+	dbBackend node.DatabaseBackend, nativeSQL bool) *HarnessTest {
 
 	t.Helper()
 
 	// Create the run context.
 	ctxt, cancel := context.WithCancel(context.Background())
 
-	manager := newNodeManager(lndBinary, dbBackend)
+	manager := newNodeManager(lndBinary, dbBackend, nativeSQL)
 
 	return &HarnessTest{
 		T:          t,

lntest/harness_node_manager.go

@@ -31,6 +31,10 @@ type nodeManager struct {
 	// dbBackend sets the database backend to use.
 	dbBackend node.DatabaseBackend
 
+	// nativeSQL sets the database backend to use native SQL when
+	// applicable.
+	nativeSQL bool
+
 	// activeNodes is a map of all running nodes, format:
 	// {pubkey: *HarnessNode}.
 	activeNodes map[uint32]*node.HarnessNode
@@ -48,12 +52,13 @@ type nodeManager struct {
 }
 
 // newNodeManager creates a new node manager instance.
-func newNodeManager(lndBinary string,
+func newNodeManager(lndBinary string, dbBackend node.DatabaseBackend,
-	dbBackend node.DatabaseBackend) *nodeManager {
+	nativeSQL bool) *nodeManager {
 
 	return &nodeManager{
 		lndBinary:    lndBinary,
 		dbBackend:    dbBackend,
+		nativeSQL:    nativeSQL,
 		activeNodes:  make(map[uint32]*node.HarnessNode),
 		standbyNodes: make(map[uint32]*node.HarnessNode),
 	}
@@ -80,6 +85,7 @@ func (nm *nodeManager) newNode(t *testing.T, name string, extraArgs []string,
 		ExtraArgs:         extraArgs,
 		FeeURL:            nm.feeServiceURL,
 		DBBackend:         nm.dbBackend,
+		NativeSQL:         nm.nativeSQL,
 		NodeID:            nm.nextNodeID(),
 		LndBinary:         nm.lndBinary,
 		NetParams:         harnessNetParams,

lntest/harness_setup.go

@@ -18,7 +18,7 @@ import (
 // 4. connect the miner and the chain backend.
 // 5. start the HarnessTest.
 func SetupHarness(t *testing.T, binaryPath, dbBackendName string,
-	feeService WebFeeService) *HarnessTest {
+	nativeSQL bool, feeService WebFeeService) *HarnessTest {
 
 	t.Log("Setting up HarnessTest...")
 
@@ -30,7 +30,7 @@ func SetupHarness(t *testing.T, binaryPath, dbBackendName string,
 	dbBackend := prepareDBBackend(t, dbBackendName)
 
 	// Create a new HarnessTest.
-	ht := NewHarnessTest(t, binaryPath, feeService, dbBackend)
+	ht := NewHarnessTest(t, binaryPath, feeService, dbBackend, nativeSQL)
 
 	// Init the miner.
 	t.Log("Prepare the miner and mine blocks to activate segwit...")

lntest/node/config.go

@@ -119,6 +119,7 @@ type BaseNodeConfig struct {
 
 	DBBackend   DatabaseBackend
 	PostgresDsn string
+	NativeSQL   bool
 
 	// NodeID is a unique ID used to identify the node.
 	NodeID uint32
@@ -277,11 +278,17 @@ func (cfg *BaseNodeConfig) GenArgs() []string {
 	case BackendPostgres:
 		args = append(args, "--db.backend=postgres")
 		args = append(args, "--db.postgres.dsn="+cfg.PostgresDsn)
+		if cfg.NativeSQL {
+			args = append(args, "--db.use-native-sql")
+		}
 
 	case BackendSqlite:
 		args = append(args, "--db.backend=sqlite")
 		args = append(args, fmt.Sprintf("--db.sqlite.busytimeout=%v",
 			wait.SqliteBusyTimeout))
+		if cfg.NativeSQL {
+			args = append(args, "--db.use-native-sql")
+		}
 	}
 
 	if cfg.FeeURL != "" {

make/testing_flags.mk

@@ -60,10 +60,16 @@ DEV_TAGS += kvdb_etcd
 endif
 
 ifeq ($(dbbackend),postgres)
+ifneq ($(nativesql),)
+ITEST_FLAGS += -nativesql
+endif
 DEV_TAGS += kvdb_postgres
 endif
 
 ifeq ($(dbbackend),sqlite)
+ifneq ($(nativesql),)
+ITEST_FLAGS += -nativesql
+endif
 DEV_TAGS += kvdb_sqlite
 endif
 

rpcserver.go

@@ -5806,7 +5806,7 @@ func (r *rpcServer) ListInvoices(ctx context.Context,
 		CreationDateEnd:   int64(req.CreationDateEnd),
 	}
 
-	invoiceSlice, err := r.server.miscDB.QueryInvoices(ctx, q)
+	invoiceSlice, err := r.server.invoicesDB.QueryInvoices(ctx, q)
 	if err != nil {
 		return nil, fmt.Errorf("unable to query invoices: %w", err)
 	}

sample-lnd.conf

@@ -1309,6 +1309,11 @@
 ; the future.
 ; db.no-rev-log-amt-data=false
 
+; If set to true, native SQL will be used instead of KV emulation for tables
+; that support it already. Note: this is an experimental feature, use at your
+; own risk.
+; db.use-native-sql=false
+
 
 [etcd]
 
@@ -1404,6 +1409,9 @@
 ; Example:
 ;   db.postgres.maxconnections=
 
+; Whether to skip executing schema migrations.
+; db.postgres.skipmigrations=false
+
 
 [sqlite]
 
@@ -1428,6 +1436,8 @@
 ;   db.sqlite.pragmaoptions=auto_vacuum=incremental
 ;   db.sqlite.pragmaoptions=temp_store=MEMORY
 
+; Whether to skip executing schema migrations.
+; db.sqlite.skipmigrations=false
 
 [bolt]
 

scripts/gen_sqlc_docker.sh

@@ -16,4 +16,4 @@ docker run \
   -e UID=$UID \
   -v "$DIR/../:/build" \
   -w /build \
-  kjconroy/sqlc:1.18.0 generate
+  sqlc/sqlc:1.25.0 generate

server.go

@@ -246,6 +246,8 @@ type server struct {
 	// channel DB that haven't been separated out yet.
 	miscDB *channeldb.DB
 
+	invoicesDB invoices.InvoiceDB
+
 	aliasMgr *aliasmgr.Manager
 
 	htlcSwitch *htlcswitch.Switch
@@ -564,6 +566,7 @@ func newServer(cfg *Config, listenAddrs []net.Addr,
 		chanStateDB:    dbs.ChanStateDB.ChannelStateDB(),
 		addrSource:     dbs.ChanStateDB,
 		miscDB:         dbs.ChanStateDB,
+		invoicesDB:     dbs.InvoiceDB,
 		cc:             cc,
 		sigPool:        lnwallet.NewSigPool(cfg.Workers.Sig, cc.Signer),
 		writePool:      writePool,

sqldb/config.go

@@ -0,0 +1,55 @@
+package sqldb
+
+import (
+	"fmt"
+	"net/url"
+	"time"
+)
+
+const (
+	// defaultMaxConns is the number of permitted active and idle
+	// connections. We want to limit this so it isn't unlimited. We use the
+	// same value for the number of idle connections as, this can speed up
+	// queries given a new connection doesn't need to be established each
+	// time.
+	defaultMaxConns = 25
+
+	// connIdleLifetime is the amount of time a connection can be idle.
+	connIdleLifetime = 5 * time.Minute
+)
+
+// SqliteConfig holds all the config arguments needed to interact with our
+// sqlite DB.
+//
+//nolint:lll
+type SqliteConfig struct {
+	Timeout        time.Duration `long:"timeout" description:"The time after which a database query should be timed out."`
+	BusyTimeout    time.Duration `long:"busytimeout" description:"The maximum amount of time to wait for a database connection to become available for a query."`
+	MaxConnections int           `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."`
+	PragmaOptions  []string      `long:"pragmaoptions" description:"A list of pragma options to set on a database connection. For example, 'auto_vacuum=incremental'. Note that the flag must be specified multiple times if multiple options are to be set."`
+	SkipMigrations bool          `long:"skipmigrations" description:"Skip applying migrations on startup."`
+}
+
+// PostgresConfig holds the postgres database configuration.
+//
+//nolint:lll
+type PostgresConfig struct {
+	Dsn            string        `long:"dsn" description:"Database connection string."`
+	Timeout        time.Duration `long:"timeout" description:"Database connection timeout. Set to zero to disable."`
+	MaxConnections int           `long:"maxconnections" description:"The maximum number of open connections to the database. Set to zero for unlimited."`
+	SkipMigrations bool          `long:"skipmigrations" description:"Skip applying migrations on startup."`
+}
+
+func (p *PostgresConfig) Validate() error {
+	if p.Dsn == "" {
+		return fmt.Errorf("DSN is required")
+	}
+
+	// Parse the DSN as a URL.
+	_, err := url.Parse(p.Dsn)
+	if err != nil {
+		return fmt.Errorf("invalid DSN: %w", err)
+	}
+
+	return nil
+}

sqldb/no_sqlite.go

@@ -0,0 +1,18 @@
+//go:build js || (windows && (arm || 386)) || (linux && (ppc64 || mips || mipsle || mips64))
+
+package sqldb
+
+import "fmt"
+
+// SqliteStore is a database store implementation that uses a sqlite backend.
+type SqliteStore struct {
+	cfg *SqliteConfig
+
+	*BaseDB
+}
+
+// NewSqliteStore attempts to open a new sqlite database based on the passed
+// config.
+func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) {
+	return nil, fmt.Errorf("SQLite backend not supported in WebAssembly")
+}

sqldb/postgres.go

@@ -2,18 +2,14 @@ package sqldb
 
 import (
 	"database/sql"
-	"fmt"
+	"net/url"
-	"testing"
+	"path"
+	"strings"
 	"time"
 
 	postgres_migrate "github.com/golang-migrate/migrate/v4/database/postgres"
 	_ "github.com/golang-migrate/migrate/v4/source/file" // Read migrations from files. // nolint:lll
 	"github.com/lightningnetwork/lnd/sqldb/sqlc"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	dsnTemplate = "postgres://%v:%v@%v:%d/%v?sslmode=%v"
 )
 
 var (
@@ -25,35 +21,45 @@ var (
 	DefaultPostgresFixtureLifetime = 10 * time.Minute
 )
 
-// PostgresConfig holds the postgres database configuration.
+// replacePasswordInDSN takes a DSN string and returns it with the password
-//
+// replaced by "***".
-//nolint:lll
+func replacePasswordInDSN(dsn string) (string, error) {
-type PostgresConfig struct {
+	// Parse the DSN as a URL
-	SkipMigrations     bool   `long:"skipmigrations" description:"Skip applying migrations on startup."`
+	u, err := url.Parse(dsn)
-	Host               string `long:"host" description:"Database server hostname."`
+	if err != nil {
-	Port               int    `long:"port" description:"Database server port."`
+		return "", err
-	User               string `long:"user" description:"Database user."`
+	}
-	Password           string `long:"password" description:"Database user's password."`
+
-	DBName             string `long:"dbname" description:"Database name to use."`
+	// Check if the URL has a user info part
-	MaxOpenConnections int    `long:"maxconnections" description:"Max open connections to keep alive to the database server."`
+	if u.User != nil {
-	RequireSSL         bool   `long:"requiressl" description:"Whether to require using SSL (mode: require) when connecting to the server."`
+		username := u.User.Username()
+
+		// Reconstruct user info with "***" as password
+		userInfo := username + ":***@"
+
+		// Rebuild the DSN with the modified user info
+		sanitizeDSN := strings.Replace(
+			dsn, u.User.String()+"@", userInfo, 1,
+		)
+
+		return sanitizeDSN, nil
+	}
+
+	// Return the original DSN if no user info is present
+	return dsn, nil
 }
 
-// DSN returns the dns to connect to the database.
+// getDatabaseNameFromDSN extracts the database name from a DSN string.
-func (s *PostgresConfig) DSN(hidePassword bool) string {
+func getDatabaseNameFromDSN(dsn string) (string, error) {
-	var sslMode = "disable"
+	// Parse the DSN as a URL
-	if s.RequireSSL {
+	u, err := url.Parse(dsn)
-		sslMode = "require"
+	if err != nil {
+		return "", err
 	}
 
-	password := s.Password
+	// The database name is the last segment of the path. Trim leading slash
-	if hidePassword {
+	// and return the last segment.
-		// Placeholder used for logging the DSN safely.
+	return path.Base(u.Path), nil
-		password = "****"
-	}
-
-	return fmt.Sprintf(dsnTemplate, s.User, password, s.Host, s.Port,
-		s.DBName, sslMode)
 }
 
 // PostgresStore is a database store implementation that uses a Postgres
@@ -67,16 +73,25 @@ type PostgresStore struct {
 // NewPostgresStore creates a new store that is backed by a Postgres database
 // backend.
 func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) {
-	log.Infof("Using SQL database '%s'", cfg.DSN(true))
+	sanitizedDSN, err := replacePasswordInDSN(cfg.Dsn)
+	if err != nil {
+		return nil, err
+	}
+	log.Infof("Using SQL database '%s'", sanitizedDSN)
 
-	rawDB, err := sql.Open("pgx", cfg.DSN(false))
+	dbName, err := getDatabaseNameFromDSN(cfg.Dsn)
+	if err != nil {
+		return nil, err
+	}
+
+	rawDB, err := sql.Open("pgx", cfg.Dsn)
 	if err != nil {
 		return nil, err
 	}
 
 	maxConns := defaultMaxConns
-	if cfg.MaxOpenConnections > 0 {
+	if cfg.MaxConnections > 0 {
-		maxConns = cfg.MaxOpenConnections
+		maxConns = cfg.MaxConnections
 	}
 
 	rawDB.SetMaxOpenConns(maxConns)
@@ -89,7 +104,7 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) {
 		// system.
 		//
 		// First, we'll need to open up a new migration instance for
-		// our current target database: sqlite.
+		// our current target database: Postgres.
 		driver, err := postgres_migrate.WithInstance(
 			rawDB, &postgres_migrate.Config{},
 		)
@@ -100,11 +115,12 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) {
 		postgresFS := newReplacerFS(sqlSchemas, map[string]string{
 			"BLOB":                "BYTEA",
 			"INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY",
+			"BIGINT PRIMARY KEY":  "BIGSERIAL PRIMARY KEY",
 			"TIMESTAMP":           "TIMESTAMP WITHOUT TIME ZONE",
 		})
 
 		err = applyMigrations(
-			postgresFS, driver, "sqlc/migrations", cfg.DBName,
+			postgresFS, driver, "sqlc/migrations", dbName,
 		)
 		if err != nil {
 			return nil, err
@@ -121,21 +137,3 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) {
 		},
 	}, nil
 }
-
-// NewTestPostgresDB is a helper function that creates a Postgres database for
-// testing.
-func NewTestPostgresDB(t *testing.T) *PostgresStore {
-	t.Helper()
-
-	t.Logf("Creating new Postgres DB for testing")
-
-	sqlFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime)
-	store, err := NewPostgresStore(sqlFixture.GetConfig())
-	require.NoError(t, err)
-
-	t.Cleanup(func() {
-		sqlFixture.TearDown(t)
-	})
-
-	return store
-}

sqldb/postgres_fixture.go

@@ -1,8 +1,12 @@
+//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) && !(netbsd || openbsd)
+
 package sqldb
 
 import (
 	"context"
+	"crypto/rand"
 	"database/sql"
+	"encoding/hex"
 	"fmt"
 	"strconv"
 	"strings"
@@ -55,6 +59,7 @@ func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture {
 			"postgres",
 			"-c", "log_statement=all",
 			"-c", "log_destination=stderr",
+			"-c", "max_connections=1000",
 		},
 	}, func(config *docker.HostConfig) {
 		// Set AutoRemove to true so that stopped container goes away
@@ -74,7 +79,7 @@ func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture {
 		host: host,
 		port: int(port),
 	}
-	databaseURL := fixture.GetDSN()
+	databaseURL := fixture.GetConfig(testPgDBName).Dsn
 	log.Infof("Connecting to Postgres fixture: %v\n", databaseURL)
 
 	// Tell docker to hard kill the container in "expiry" seconds.
@@ -103,20 +108,13 @@ func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture {
 	return fixture
 }
 
-// GetDSN returns the DSN (Data Source Name) for the started Postgres node.
-func (f *TestPgFixture) GetDSN() string {
-	return f.GetConfig().DSN(false)
-}
-
 // GetConfig returns the full config of the Postgres node.
-func (f *TestPgFixture) GetConfig() *PostgresConfig {
+func (f *TestPgFixture) GetConfig(dbName string) *PostgresConfig {
 	return &PostgresConfig{
-		Host:       f.host,
+		Dsn: fmt.Sprintf(
-		Port:       f.port,
+			"postgres://%v:%v@%v:%v/%v?sslmode=disable",
-		User:       testPgUser,
+			testPgUser, testPgPass, f.host, f.port, dbName,
-		Password:   testPgPass,
+		),
-		DBName:     testPgDBName,
-		RequireSSL: false,
 	}
 }
 
@@ -126,15 +124,32 @@ func (f *TestPgFixture) TearDown(t *testing.T) {
 	require.NoError(t, err, "Could not purge resource")
 }
 
-// ClearDB clears the database.
+// NewTestPostgresDB is a helper function that creates a Postgres database for
-func (f *TestPgFixture) ClearDB(t *testing.T) {
+// testing using the given fixture.
-	dbConn, err := sql.Open("postgres", f.GetDSN())
+func NewTestPostgresDB(t *testing.T, fixture *TestPgFixture) *PostgresStore {
+	t.Helper()
+
+	// Create random database name.
+	randBytes := make([]byte, 8)
+	_, err := rand.Read(randBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dbName := "test_" + hex.EncodeToString(randBytes)
+
+	t.Logf("Creating new Postgres DB '%s' for testing", dbName)
+
+	_, err = fixture.db.ExecContext(
+		context.Background(), "CREATE DATABASE "+dbName,
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cfg := fixture.GetConfig(dbName)
+	store, err := NewPostgresStore(cfg)
 	require.NoError(t, err)
 
-	_, err = dbConn.ExecContext(
+	return store
-		context.Background(),
-		`DROP SCHEMA IF EXISTS public CASCADE;
-		 CREATE SCHEMA public;`,
-	)
-	require.NoError(t, err)
 }

sqldb/sqlc/amp_invoices.sql.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.18.0
+//   sqlc v1.25.0
 // source: amp_invoices.sql
 
 package sqlc
@@ -11,67 +11,69 @@ import (
 	"time"
 )
 
-const deleteAMPHTLCCustomRecords = `-- name: DeleteAMPHTLCCustomRecords :exec
+const fetchAMPSubInvoiceHTLCs = `-- name: FetchAMPSubInvoiceHTLCs :many
-WITH htlc_ids AS (
+SELECT 
-    SELECT htlc_id
+    amp.set_id, amp.root_share, amp.child_index, amp.hash, amp.preimage, 
-    FROM amp_invoice_htlcs 
+    invoice_htlcs.id, invoice_htlcs.chan_id, invoice_htlcs.htlc_id, invoice_htlcs.amount_msat, invoice_htlcs.total_mpp_msat, invoice_htlcs.accept_height, invoice_htlcs.accept_time, invoice_htlcs.expiry_height, invoice_htlcs.state, invoice_htlcs.resolve_time, invoice_htlcs.invoice_id
-    WHERE invoice_id = $1
+FROM amp_sub_invoice_htlcs amp
+INNER JOIN invoice_htlcs ON amp.htlc_id = invoice_htlcs.id
+WHERE amp.invoice_id = $1
+AND (
+    set_id = $2 OR 
+    $2 IS NULL
 )
-DELETE
-FROM invoice_htlc_custom_records
-WHERE htlc_id IN (SELECT id FROM htlc_ids)
 `
 
-func (q *Queries) DeleteAMPHTLCCustomRecords(ctx context.Context, invoiceID int32) error {
+type FetchAMPSubInvoiceHTLCsParams struct {
-	_, err := q.db.ExecContext(ctx, deleteAMPHTLCCustomRecords, invoiceID)
+	InvoiceID int64
-	return err
+	SetID     []byte
 }
 
-const deleteAMPHTLCs = `-- name: DeleteAMPHTLCs :exec
+type FetchAMPSubInvoiceHTLCsRow struct {
-DELETE 
+	SetID        []byte
-FROM amp_invoice_htlcs  
+	RootShare    []byte
-WHERE invoice_id = $1
+	ChildIndex   int64
-`
+	Hash         []byte
-
+	Preimage     []byte
-func (q *Queries) DeleteAMPHTLCs(ctx context.Context, invoiceID int32) error {
+	ID           int64
-	_, err := q.db.ExecContext(ctx, deleteAMPHTLCs, invoiceID)
+	ChanID       string
-	return err
+	HtlcID       int64
+	AmountMsat   int64
+	TotalMppMsat sql.NullInt64
+	AcceptHeight int32
+	AcceptTime   time.Time
+	ExpiryHeight int32
+	State        int16
+	ResolveTime  sql.NullTime
+	InvoiceID    int64
 }
 
-const deleteAMPInvoiceHTLC = `-- name: DeleteAMPInvoiceHTLC :exec
+func (q *Queries) FetchAMPSubInvoiceHTLCs(ctx context.Context, arg FetchAMPSubInvoiceHTLCsParams) ([]FetchAMPSubInvoiceHTLCsRow, error) {
-DELETE 
+	rows, err := q.db.QueryContext(ctx, fetchAMPSubInvoiceHTLCs, arg.InvoiceID, arg.SetID)
-FROM amp_invoice_htlcs
-WHERE set_id = $1
-`
-
-func (q *Queries) DeleteAMPInvoiceHTLC(ctx context.Context, setID []byte) error {
-	_, err := q.db.ExecContext(ctx, deleteAMPInvoiceHTLC, setID)
-	return err
-}
-
-const getAMPInvoiceHTLCsByInvoiceID = `-- name: GetAMPInvoiceHTLCsByInvoiceID :many
-SELECT set_id, htlc_id, invoice_id, root_share, child_index, hash, preimage
-FROM amp_invoice_htlcs
-WHERE invoice_id = $1
-`
-
-func (q *Queries) GetAMPInvoiceHTLCsByInvoiceID(ctx context.Context, invoiceID int32) ([]AmpInvoiceHtlc, error) {
-	rows, err := q.db.QueryContext(ctx, getAMPInvoiceHTLCsByInvoiceID, invoiceID)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
-	var items []AmpInvoiceHtlc
+	var items []FetchAMPSubInvoiceHTLCsRow
 	for rows.Next() {
-		var i AmpInvoiceHtlc
+		var i FetchAMPSubInvoiceHTLCsRow
 		if err := rows.Scan(
 			&i.SetID,
-			&i.HtlcID,
-			&i.InvoiceID,
 			&i.RootShare,
 			&i.ChildIndex,
 			&i.Hash,
 			&i.Preimage,
+			&i.ID,
+			&i.ChanID,
+			&i.HtlcID,
+			&i.AmountMsat,
+			&i.TotalMppMsat,
+			&i.AcceptHeight,
+			&i.AcceptTime,
+			&i.ExpiryHeight,
+			&i.State,
+			&i.ResolveTime,
+			&i.InvoiceID,
 		); err != nil {
 			return nil, err
 		}
@@ -86,29 +88,37 @@ func (q *Queries) GetAMPInvoiceHTLCsByInvoiceID(ctx context.Context, invoiceID i
 	return items, nil
 }
 
-const getAMPInvoiceHTLCsBySetID = `-- name: GetAMPInvoiceHTLCsBySetID :many
+const fetchAMPSubInvoices = `-- name: FetchAMPSubInvoices :many
-SELECT set_id, htlc_id, invoice_id, root_share, child_index, hash, preimage
+SELECT set_id, state, created_at, settled_at, settle_index, invoice_id
-FROM amp_invoice_htlcs
+FROM amp_sub_invoices
-WHERE set_id = $1
+WHERE invoice_id = $1 
+AND (
+    set_id = $2 OR 
+    $2 IS NULL
+)
 `
 
-func (q *Queries) GetAMPInvoiceHTLCsBySetID(ctx context.Context, setID []byte) ([]AmpInvoiceHtlc, error) {
+type FetchAMPSubInvoicesParams struct {
-	rows, err := q.db.QueryContext(ctx, getAMPInvoiceHTLCsBySetID, setID)
+	InvoiceID int64
+	SetID     []byte
+}
+
+func (q *Queries) FetchAMPSubInvoices(ctx context.Context, arg FetchAMPSubInvoicesParams) ([]AmpSubInvoice, error) {
+	rows, err := q.db.QueryContext(ctx, fetchAMPSubInvoices, arg.InvoiceID, arg.SetID)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
-	var items []AmpInvoiceHtlc
+	var items []AmpSubInvoice
 	for rows.Next() {
-		var i AmpInvoiceHtlc
+		var i AmpSubInvoice
 		if err := rows.Scan(
 			&i.SetID,
-			&i.HtlcID,
+			&i.State,
+			&i.CreatedAt,
+			&i.SettledAt,
+			&i.SettleIndex,
 			&i.InvoiceID,
-			&i.RootShare,
-			&i.ChildIndex,
-			&i.Hash,
-			&i.Preimage,
 		); err != nil {
 			return nil, err
 		}
@@ -123,28 +133,84 @@ func (q *Queries) GetAMPInvoiceHTLCsBySetID(ctx context.Context, setID []byte) (
 	return items, nil
 }
 
-const getSetIDHTLCsCustomRecords = `-- name: GetSetIDHTLCsCustomRecords :many
+const fetchSettledAMPSubInvoices = `-- name: FetchSettledAMPSubInvoices :many
-SELECT ihcr.htlc_id, key, value
+SELECT 
-FROM amp_invoice_htlcs aih JOIN invoice_htlc_custom_records ihcr ON aih.id=ihcr.htlc_id 
+    a.set_id, 
-WHERE aih.set_id = $1
+    a.settle_index as amp_settle_index, 
+    a.settled_at as amp_settled_at,
+    i.id, i.hash, i.preimage, i.settle_index, i.settled_at, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.payment_request_hash, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at
+FROM amp_sub_invoices a
+INNER JOIN invoices i ON a.invoice_id = i.id
+WHERE (
+    a.settle_index >= $1 OR
+    $1 IS NULL
+) AND (
+    a.settle_index <= $2 OR
+    $2 IS NULL
+)
 `
 
-type GetSetIDHTLCsCustomRecordsRow struct {
+type FetchSettledAMPSubInvoicesParams struct {
-	HtlcID int64
+	SettleIndexGet sql.NullInt64
-	Key    int64
+	SettleIndexLet sql.NullInt64
-	Value  []byte
 }
 
-func (q *Queries) GetSetIDHTLCsCustomRecords(ctx context.Context, setID []byte) ([]GetSetIDHTLCsCustomRecordsRow, error) {
+type FetchSettledAMPSubInvoicesRow struct {
-	rows, err := q.db.QueryContext(ctx, getSetIDHTLCsCustomRecords, setID)
+	SetID              []byte
+	AmpSettleIndex     sql.NullInt64
+	AmpSettledAt       sql.NullTime
+	ID                 int64
+	Hash               []byte
+	Preimage           []byte
+	SettleIndex        sql.NullInt64
+	SettledAt          sql.NullTime
+	Memo               sql.NullString
+	AmountMsat         int64
+	CltvDelta          sql.NullInt32
+	Expiry             int32
+	PaymentAddr        []byte
+	PaymentRequest     sql.NullString
+	PaymentRequestHash []byte
+	State              int16
+	AmountPaidMsat     int64
+	IsAmp              bool
+	IsHodl             bool
+	IsKeysend          bool
+	CreatedAt          time.Time
+}
+
+func (q *Queries) FetchSettledAMPSubInvoices(ctx context.Context, arg FetchSettledAMPSubInvoicesParams) ([]FetchSettledAMPSubInvoicesRow, error) {
+	rows, err := q.db.QueryContext(ctx, fetchSettledAMPSubInvoices, arg.SettleIndexGet, arg.SettleIndexLet)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
-	var items []GetSetIDHTLCsCustomRecordsRow
+	var items []FetchSettledAMPSubInvoicesRow
 	for rows.Next() {
-		var i GetSetIDHTLCsCustomRecordsRow
+		var i FetchSettledAMPSubInvoicesRow
-		if err := rows.Scan(&i.HtlcID, &i.Key, &i.Value); err != nil {
+		if err := rows.Scan(
+			&i.SetID,
+			&i.AmpSettleIndex,
+			&i.AmpSettledAt,
+			&i.ID,
+			&i.Hash,
+			&i.Preimage,
+			&i.SettleIndex,
+			&i.SettledAt,
+			&i.Memo,
+			&i.AmountMsat,
+			&i.CltvDelta,
+			&i.Expiry,
+			&i.PaymentAddr,
+			&i.PaymentRequest,
+			&i.PaymentRequestHash,
+			&i.State,
+			&i.AmountPaidMsat,
+			&i.IsAmp,
+			&i.IsHodl,
+			&i.IsKeysend,
+			&i.CreatedAt,
+		); err != nil {
 			return nil, err
 		}
 		items = append(items, i)
@@ -158,15 +224,27 @@ func (q *Queries) GetSetIDHTLCsCustomRecords(ctx context.Context, setID []byte)
 	return items, nil
 }
 
-const insertAMPInvoiceHTLC = `-- name: InsertAMPInvoiceHTLC :exec
+const getAMPInvoiceID = `-- name: GetAMPInvoiceID :one
-INSERT INTO amp_invoice_htlcs (
+SELECT invoice_id FROM amp_sub_invoices WHERE set_id = $1
-    set_id, htlc_id, root_share, child_index, hash, preimage
+`
+
+func (q *Queries) GetAMPInvoiceID(ctx context.Context, setID []byte) (int64, error) {
+	row := q.db.QueryRowContext(ctx, getAMPInvoiceID, setID)
+	var invoice_id int64
+	err := row.Scan(&invoice_id)
+	return invoice_id, err
+}
+
+const insertAMPSubInvoiceHTLC = `-- name: InsertAMPSubInvoiceHTLC :exec
+INSERT INTO amp_sub_invoice_htlcs (
+    invoice_id, set_id, htlc_id, root_share, child_index, hash, preimage
 ) VALUES (
-    $1, $2, $3, $4, $5, $6
+    $1, $2, $3, $4, $5, $6, $7
 )
 `
 
-type InsertAMPInvoiceHTLCParams struct {
+type InsertAMPSubInvoiceHTLCParams struct {
+	InvoiceID  int64
 	SetID      []byte
 	HtlcID     int64
 	RootShare  []byte
@@ -175,8 +253,9 @@ type InsertAMPInvoiceHTLCParams struct {
 	Preimage   []byte
 }
 
-func (q *Queries) InsertAMPInvoiceHTLC(ctx context.Context, arg InsertAMPInvoiceHTLCParams) error {
+func (q *Queries) InsertAMPSubInvoiceHTLC(ctx context.Context, arg InsertAMPSubInvoiceHTLCParams) error {
-	_, err := q.db.ExecContext(ctx, insertAMPInvoiceHTLC,
+	_, err := q.db.ExecContext(ctx, insertAMPSubInvoiceHTLC,
+		arg.InvoiceID,
 		arg.SetID,
 		arg.HtlcID,
 		arg.RootShare,
@@ -187,140 +266,75 @@ func (q *Queries) InsertAMPInvoiceHTLC(ctx context.Context, arg InsertAMPInvoice
 	return err
 }
 
-const insertAMPInvoicePayment = `-- name: InsertAMPInvoicePayment :exec
+const updateAMPSubInvoiceHTLCPreimage = `-- name: UpdateAMPSubInvoiceHTLCPreimage :execresult
-INSERT INTO amp_invoice_payments (
+UPDATE amp_sub_invoice_htlcs AS a
-    set_id, state, created_at, settled_index, invoice_id
+SET preimage = $4
-) VALUES (
+WHERE a.invoice_id = $1 AND a.set_id = $2 AND a.htlc_id = (
-    $1, $2, $3, $4, $5
+    SELECT id FROM invoice_htlcs AS i WHERE i.htlc_id = $3
 )
 `
 
-type InsertAMPInvoicePaymentParams struct {
+type UpdateAMPSubInvoiceHTLCPreimageParams struct {
-	SetID        []byte
+	InvoiceID int64
-	State        int16
+	SetID     []byte
-	CreatedAt    time.Time
+	HtlcID    int64
-	SettledIndex sql.NullInt32
+	Preimage  []byte
-	InvoiceID    int32
 }
 
-func (q *Queries) InsertAMPInvoicePayment(ctx context.Context, arg InsertAMPInvoicePaymentParams) error {
+func (q *Queries) UpdateAMPSubInvoiceHTLCPreimage(ctx context.Context, arg UpdateAMPSubInvoiceHTLCPreimageParams) (sql.Result, error) {
-	_, err := q.db.ExecContext(ctx, insertAMPInvoicePayment,
+	return q.db.ExecContext(ctx, updateAMPSubInvoiceHTLCPreimage,
+		arg.InvoiceID,
+		arg.SetID,
+		arg.HtlcID,
+		arg.Preimage,
+	)
+}
+
+const updateAMPSubInvoiceState = `-- name: UpdateAMPSubInvoiceState :exec
+UPDATE amp_sub_invoices
+SET state = $2, 
+    settle_index = COALESCE(settle_index, $3),
+    settled_at = COALESCE(settled_at, $4)
+WHERE set_id = $1
+`
+
+type UpdateAMPSubInvoiceStateParams struct {
+	SetID       []byte
+	State       int16
+	SettleIndex sql.NullInt64
+	SettledAt   sql.NullTime
+}
+
+func (q *Queries) UpdateAMPSubInvoiceState(ctx context.Context, arg UpdateAMPSubInvoiceStateParams) error {
+	_, err := q.db.ExecContext(ctx, updateAMPSubInvoiceState,
+		arg.SetID,
+		arg.State,
+		arg.SettleIndex,
+		arg.SettledAt,
+	)
+	return err
+}
+
+const upsertAMPSubInvoice = `-- name: UpsertAMPSubInvoice :execresult
+INSERT INTO amp_sub_invoices (
+    set_id, state, created_at, invoice_id
+) VALUES (
+    $1, $2, $3, $4
+) ON CONFLICT (set_id, invoice_id) DO NOTHING
+`
+
+type UpsertAMPSubInvoiceParams struct {
+	SetID     []byte
+	State     int16
+	CreatedAt time.Time
+	InvoiceID int64
+}
+
+func (q *Queries) UpsertAMPSubInvoice(ctx context.Context, arg UpsertAMPSubInvoiceParams) (sql.Result, error) {
+	return q.db.ExecContext(ctx, upsertAMPSubInvoice,
 		arg.SetID,
 		arg.State,
 		arg.CreatedAt,
-		arg.SettledIndex,
 		arg.InvoiceID,
 	)
-	return err
-}
-
-const selectAMPInvoicePayments = `-- name: SelectAMPInvoicePayments :many
-SELECT aip.set_id, aip.state, aip.created_at, aip.settled_index, aip.invoice_id, ip.id, ip.settled_at, ip.amount_paid_msat, ip.invoice_id
-FROM amp_invoice_payments aip LEFT JOIN invoice_payments ip ON aip.settled_index = ip.id
-WHERE (
-    set_id = $1 OR 
-    $1 IS NULL
-) AND (
-    aip.settled_index = $2 OR 
-    $2 IS NULL
-) AND (
-    aip.invoice_id = $3 OR 
-    $3 IS NULL
-)
-`
-
-type SelectAMPInvoicePaymentsParams struct {
-	SetID        []byte
-	SettledIndex sql.NullInt32
-	InvoiceID    sql.NullInt32
-}
-
-type SelectAMPInvoicePaymentsRow struct {
-	SetID          []byte
-	State          int16
-	CreatedAt      time.Time
-	SettledIndex   sql.NullInt32
-	InvoiceID      int32
-	ID             sql.NullInt32
-	SettledAt      sql.NullTime
-	AmountPaidMsat sql.NullInt64
-	InvoiceID_2    sql.NullInt32
-}
-
-func (q *Queries) SelectAMPInvoicePayments(ctx context.Context, arg SelectAMPInvoicePaymentsParams) ([]SelectAMPInvoicePaymentsRow, error) {
-	rows, err := q.db.QueryContext(ctx, selectAMPInvoicePayments, arg.SetID, arg.SettledIndex, arg.InvoiceID)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []SelectAMPInvoicePaymentsRow
-	for rows.Next() {
-		var i SelectAMPInvoicePaymentsRow
-		if err := rows.Scan(
-			&i.SetID,
-			&i.State,
-			&i.CreatedAt,
-			&i.SettledIndex,
-			&i.InvoiceID,
-			&i.ID,
-			&i.SettledAt,
-			&i.AmountPaidMsat,
-			&i.InvoiceID_2,
-		); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Close(); err != nil {
-		return nil, err
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
-}
-
-const updateAMPInvoiceHTLC = `-- name: UpdateAMPInvoiceHTLC :exec
-UPDATE amp_invoice_htlcs
-SET preimage = $1
-WHERE htlc_id = $2
-`
-
-type UpdateAMPInvoiceHTLCParams struct {
-	Preimage []byte
-	HtlcID   int64
-}
-
-func (q *Queries) UpdateAMPInvoiceHTLC(ctx context.Context, arg UpdateAMPInvoiceHTLCParams) error {
-	_, err := q.db.ExecContext(ctx, updateAMPInvoiceHTLC, arg.Preimage, arg.HtlcID)
-	return err
-}
-
-const updateAMPPayment = `-- name: UpdateAMPPayment :exec
-UPDATE amp_invoice_payments
-SET state = $1, settled_index = $2
-WHERE state = 0 AND (
-    set_id = $3 OR 
-    $3 IS NULL
-) AND (
-    invoice_id = $4 OR 
-    $4 IS NULL
-)
-`
-
-type UpdateAMPPaymentParams struct {
-	State        int16
-	SettledIndex sql.NullInt32
-	SetID        []byte
-	InvoiceID    sql.NullInt32
-}
-
-func (q *Queries) UpdateAMPPayment(ctx context.Context, arg UpdateAMPPaymentParams) error {
-	_, err := q.db.ExecContext(ctx, updateAMPPayment,
-		arg.State,
-		arg.SettledIndex,
-		arg.SetID,
-		arg.InvoiceID,
-	)
-	return err
 }

sqldb/sqlc/db.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.18.0
+//   sqlc v1.25.0
 
 package sqlc
 

sqldb/sqlc/invoice_events.sql.go

@@ -1,128 +1,122 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.18.0
+//   sqlc v1.25.0
 // source: invoice_events.sql
 
 package sqlc
 
 import (
 	"context"
-	"database/sql"
 	"time"
 )
 
-const deleteInvoiceEvents = `-- name: DeleteInvoiceEvents :exec
+const onAMPSubInvoiceCanceled = `-- name: OnAMPSubInvoiceCanceled :exec
-DELETE
-FROM invoice_events
-WHERE invoice_id = $1
-`
-
-func (q *Queries) DeleteInvoiceEvents(ctx context.Context, invoiceID int32) error {
-	_, err := q.db.ExecContext(ctx, deleteInvoiceEvents, invoiceID)
-	return err
-}
-
-const insertInvoiceEvent = `-- name: InsertInvoiceEvent :exec
 INSERT INTO invoice_events (
-    created_at, invoice_id, htlc_id, set_id, event_type, event_metadata
+    added_at, event_type, invoice_id, set_id
 ) VALUES (
-    $1, $2, $3, $4, $5, $6
+    $1, 4, $2, $3
 )
 `
 
-type InsertInvoiceEventParams struct {
+type OnAMPSubInvoiceCanceledParams struct {
-	CreatedAt     time.Time
+	AddedAt   time.Time
-	InvoiceID     int32
+	InvoiceID int64
-	HtlcID        sql.NullInt64
+	SetID     []byte
-	SetID         []byte
-	EventType     int32
-	EventMetadata []byte
 }
 
-func (q *Queries) InsertInvoiceEvent(ctx context.Context, arg InsertInvoiceEventParams) error {
+func (q *Queries) OnAMPSubInvoiceCanceled(ctx context.Context, arg OnAMPSubInvoiceCanceledParams) error {
-	_, err := q.db.ExecContext(ctx, insertInvoiceEvent,
+	_, err := q.db.ExecContext(ctx, onAMPSubInvoiceCanceled, arg.AddedAt, arg.InvoiceID, arg.SetID)
-		arg.CreatedAt,
-		arg.InvoiceID,
-		arg.HtlcID,
-		arg.SetID,
-		arg.EventType,
-		arg.EventMetadata,
-	)
 	return err
 }
 
-const selectInvoiceEvents = `-- name: SelectInvoiceEvents :many
+const onAMPSubInvoiceCreated = `-- name: OnAMPSubInvoiceCreated :exec
-SELECT id, created_at, invoice_id, htlc_id, set_id, event_type, event_metadata
+INSERT INTO invoice_events (
-FROM invoice_events
+    added_at, event_type, invoice_id, set_id
-WHERE (
+) VALUES (
-    invoice_id = $1 OR 
+    $1, 3, $2, $3
-    $1 IS NULL
+)
-) AND (
-    htlc_id = $2 OR 
-    $2 IS NULL
-) AND (
-    set_id = $3 OR 
-    $3 IS NULL
-) AND (
-    event_type = $4 OR 
-    $4 IS NULL
-) AND (
-    created_at >= $5 OR 
-    $5 IS NULL
-) AND (
-    created_at <= $6 OR 
-    $6 IS NULL
-) 
-LIMIT $8 OFFSET $7
 `
 
-type SelectInvoiceEventsParams struct {
+type OnAMPSubInvoiceCreatedParams struct {
-	InvoiceID     sql.NullInt32
+	AddedAt   time.Time
-	HtlcID        sql.NullInt64
+	InvoiceID int64
-	SetID         []byte
+	SetID     []byte
-	EventType     sql.NullInt32
-	CreatedAfter  sql.NullTime
-	CreatedBefore sql.NullTime
-	NumOffset     int32
-	NumLimit      int32
 }
 
-func (q *Queries) SelectInvoiceEvents(ctx context.Context, arg SelectInvoiceEventsParams) ([]InvoiceEvent, error) {
+func (q *Queries) OnAMPSubInvoiceCreated(ctx context.Context, arg OnAMPSubInvoiceCreatedParams) error {
-	rows, err := q.db.QueryContext(ctx, selectInvoiceEvents,
+	_, err := q.db.ExecContext(ctx, onAMPSubInvoiceCreated, arg.AddedAt, arg.InvoiceID, arg.SetID)
-		arg.InvoiceID,
+	return err
-		arg.HtlcID,
+}
-		arg.SetID,
+
-		arg.EventType,
+const onAMPSubInvoiceSettled = `-- name: OnAMPSubInvoiceSettled :exec
-		arg.CreatedAfter,
+INSERT INTO invoice_events (
-		arg.CreatedBefore,
+    added_at, event_type, invoice_id, set_id
-		arg.NumOffset,
+) VALUES (
-		arg.NumLimit,
+    $1, 5, $2, $3
-	)
+)
-	if err != nil {
+`
-		return nil, err
+
-	}
+type OnAMPSubInvoiceSettledParams struct {
-	defer rows.Close()
+	AddedAt   time.Time
-	var items []InvoiceEvent
+	InvoiceID int64
-	for rows.Next() {
+	SetID     []byte
-		var i InvoiceEvent
+}
-		if err := rows.Scan(
+
-			&i.ID,
+func (q *Queries) OnAMPSubInvoiceSettled(ctx context.Context, arg OnAMPSubInvoiceSettledParams) error {
-			&i.CreatedAt,
+	_, err := q.db.ExecContext(ctx, onAMPSubInvoiceSettled, arg.AddedAt, arg.InvoiceID, arg.SetID)
-			&i.InvoiceID,
+	return err
-			&i.HtlcID,
+}
-			&i.SetID,
+
-			&i.EventType,
+const onInvoiceCanceled = `-- name: OnInvoiceCanceled :exec
-			&i.EventMetadata,
+INSERT INTO invoice_events (
-		); err != nil {
+    added_at, event_type, invoice_id
-			return nil, err
+) VALUES (
-		}
+    $1, 1, $2
-		items = append(items, i)
+)
-	}
+`
-	if err := rows.Close(); err != nil {
+
-		return nil, err
+type OnInvoiceCanceledParams struct {
-	}
+	AddedAt   time.Time
-	if err := rows.Err(); err != nil {
+	InvoiceID int64
-		return nil, err
+}
-	}
+
-	return items, nil
+func (q *Queries) OnInvoiceCanceled(ctx context.Context, arg OnInvoiceCanceledParams) error {
+	_, err := q.db.ExecContext(ctx, onInvoiceCanceled, arg.AddedAt, arg.InvoiceID)
+	return err
+}
+
+const onInvoiceCreated = `-- name: OnInvoiceCreated :exec
+INSERT INTO invoice_events (
+    added_at, event_type, invoice_id
+) VALUES (
+    $1, 0, $2
+)
+`
+
+type OnInvoiceCreatedParams struct {
+	AddedAt   time.Time
+	InvoiceID int64
+}
+
+func (q *Queries) OnInvoiceCreated(ctx context.Context, arg OnInvoiceCreatedParams) error {
+	_, err := q.db.ExecContext(ctx, onInvoiceCreated, arg.AddedAt, arg.InvoiceID)
+	return err
+}
+
+const onInvoiceSettled = `-- name: OnInvoiceSettled :exec
+INSERT INTO invoice_events (
+    added_at, event_type, invoice_id
+) VALUES (
+    $1, 2, $2
+)
+`
+
+type OnInvoiceSettledParams struct {
+	AddedAt   time.Time
+	InvoiceID int64
+}
+
+func (q *Queries) OnInvoiceSettled(ctx context.Context, arg OnInvoiceSettledParams) error {
+	_, err := q.db.ExecContext(ctx, onInvoiceSettled, arg.AddedAt, arg.InvoiceID)
+	return err
 }

sqldb/sqlc/invoices.sql.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.18.0
+//   sqlc v1.25.0
 // source: invoices.sql
 
 package sqlc
@@ -11,7 +11,17 @@ import (
 	"time"
 )
 
-const deleteInvoice = `-- name: DeleteInvoice :exec
+const deleteCanceledInvoices = `-- name: DeleteCanceledInvoices :execresult
+DELETE
+FROM invoices
+WHERE state = 2
+`
+
+func (q *Queries) DeleteCanceledInvoices(ctx context.Context) (sql.Result, error) {
+	return q.db.ExecContext(ctx, deleteCanceledInvoices)
+}
+
+const deleteInvoice = `-- name: DeleteInvoice :execresult
 DELETE 
 FROM invoices 
 WHERE (
@@ -21,7 +31,7 @@ WHERE (
     hash = $2 OR 
     $2 IS NULL
 ) AND (
-    preimage = $3 OR 
+    settle_index = $3 OR 
     $3 IS NULL
 ) AND (
     payment_addr = $4 OR
@@ -30,174 +40,24 @@ WHERE (
 `
 
 type DeleteInvoiceParams struct {
-	AddIndex    sql.NullInt32
+	AddIndex    sql.NullInt64
 	Hash        []byte
-	Preimage    []byte
+	SettleIndex sql.NullInt64
 	PaymentAddr []byte
 }
 
-func (q *Queries) DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) error {
+func (q *Queries) DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) (sql.Result, error) {
-	_, err := q.db.ExecContext(ctx, deleteInvoice,
+	return q.db.ExecContext(ctx, deleteInvoice,
 		arg.AddIndex,
 		arg.Hash,
-		arg.Preimage,
+		arg.SettleIndex,
 		arg.PaymentAddr,
 	)
-	return err
-}
-
-const deleteInvoiceFeatures = `-- name: DeleteInvoiceFeatures :exec
-DELETE 
-FROM invoice_features
-WHERE invoice_id = $1
-`
-
-func (q *Queries) DeleteInvoiceFeatures(ctx context.Context, invoiceID int32) error {
-	_, err := q.db.ExecContext(ctx, deleteInvoiceFeatures, invoiceID)
-	return err
-}
-
-const deleteInvoiceHTLC = `-- name: DeleteInvoiceHTLC :exec
-DELETE
-FROM invoice_htlcs
-WHERE htlc_id = $1
-`
-
-func (q *Queries) DeleteInvoiceHTLC(ctx context.Context, htlcID int64) error {
-	_, err := q.db.ExecContext(ctx, deleteInvoiceHTLC, htlcID)
-	return err
-}
-
-const deleteInvoiceHTLCCustomRecords = `-- name: DeleteInvoiceHTLCCustomRecords :exec
-WITH htlc_ids AS (
-    SELECT ih.id
-    FROM invoice_htlcs ih JOIN invoice_htlc_custom_records ihcr ON ih.id=ihcr.htlc_id 
-    WHERE ih.invoice_id = $1
-)
-DELETE
-FROM invoice_htlc_custom_records
-WHERE htlc_id IN (SELECT id FROM htlc_ids)
-`
-
-func (q *Queries) DeleteInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) error {
-	_, err := q.db.ExecContext(ctx, deleteInvoiceHTLCCustomRecords, invoiceID)
-	return err
-}
-
-const deleteInvoiceHTLCs = `-- name: DeleteInvoiceHTLCs :exec
-DELETE
-FROM invoice_htlcs
-WHERE invoice_id = $1
-`
-
-func (q *Queries) DeleteInvoiceHTLCs(ctx context.Context, invoiceID int32) error {
-	_, err := q.db.ExecContext(ctx, deleteInvoiceHTLCs, invoiceID)
-	return err
-}
-
-const filterInvoicePayments = `-- name: FilterInvoicePayments :many
-SELECT  
-    ip.id AS settle_index, ip.amount_paid_msat, ip.settled_at AS settle_date, 
-    i.id, i.hash, i.preimage, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at 
-FROM invoice_payments ip JOIN invoices i ON ip.invoice_id = i.id
-WHERE (
-    ip.id >= $1 OR 
-    $1 IS NULL
-) AND (
-    ip.settled_at >= $2 OR 
-    $2 IS NULL
-)
-ORDER BY
-    CASE
-        WHEN $3 = FALSE THEN ip.id  
-        ELSE NULL
-    END ASC,
-    CASE
-        WHEN $3 = TRUE THEN ip.id  
-        ELSE NULL
-    END DESC
-LIMIT $5 OFFSET $4
-`
-
-type FilterInvoicePaymentsParams struct {
-	SettleIndexGet sql.NullInt32
-	SettledAfter   sql.NullTime
-	Reverse        interface{}
-	NumOffset      int32
-	NumLimit       int32
-}
-
-type FilterInvoicePaymentsRow struct {
-	SettleIndex      int32
-	AmountPaidMsat   int64
-	SettleDate       time.Time
-	ID               int32
-	Hash             []byte
-	Preimage         []byte
-	Memo             sql.NullString
-	AmountMsat       int64
-	CltvDelta        sql.NullInt32
-	Expiry           int32
-	PaymentAddr      []byte
-	PaymentRequest   sql.NullString
-	State            int16
-	AmountPaidMsat_2 int64
-	IsAmp            bool
-	IsHodl           bool
-	IsKeysend        bool
-	CreatedAt        time.Time
-}
-
-func (q *Queries) FilterInvoicePayments(ctx context.Context, arg FilterInvoicePaymentsParams) ([]FilterInvoicePaymentsRow, error) {
-	rows, err := q.db.QueryContext(ctx, filterInvoicePayments,
-		arg.SettleIndexGet,
-		arg.SettledAfter,
-		arg.Reverse,
-		arg.NumOffset,
-		arg.NumLimit,
-	)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []FilterInvoicePaymentsRow
-	for rows.Next() {
-		var i FilterInvoicePaymentsRow
-		if err := rows.Scan(
-			&i.SettleIndex,
-			&i.AmountPaidMsat,
-			&i.SettleDate,
-			&i.ID,
-			&i.Hash,
-			&i.Preimage,
-			&i.Memo,
-			&i.AmountMsat,
-			&i.CltvDelta,
-			&i.Expiry,
-			&i.PaymentAddr,
-			&i.PaymentRequest,
-			&i.State,
-			&i.AmountPaidMsat_2,
-			&i.IsAmp,
-			&i.IsHodl,
-			&i.IsKeysend,
-			&i.CreatedAt,
-		); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Close(); err != nil {
-		return nil, err
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
 }
 
 const filterInvoices = `-- name: FilterInvoices :many
-SELECT id, hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, created_at 
+SELECT
+    invoices.id, invoices.hash, invoices.preimage, invoices.settle_index, invoices.settled_at, invoices.memo, invoices.amount_msat, invoices.cltv_delta, invoices.expiry, invoices.payment_addr, invoices.payment_request, invoices.payment_request_hash, invoices.state, invoices.amount_paid_msat, invoices.is_amp, invoices.is_hodl, invoices.is_keysend, invoices.created_at
 FROM invoices
 WHERE (
     id >= $1 OR 
@@ -206,48 +66,58 @@ WHERE (
     id <= $2 OR 
     $2 IS NULL
 ) AND (
-    state = $3 OR 
+    settle_index >= $3 OR
     $3 IS NULL
 ) AND (
-    created_at >= $4 OR
+    settle_index <= $4 OR
     $4 IS NULL
 ) AND (
-    created_at <= $5 OR 
+    state = $5 OR 
     $5 IS NULL
+) AND (
+    created_at >= $6 OR
+    $6 IS NULL
+) AND (
+    created_at <= $7 OR 
+    $7 IS NULL
 ) AND (
     CASE
-        WHEN $6=TRUE THEN (state = 0 OR state = 3)
+        WHEN $8=TRUE THEN (state = 0 OR state = 3)
         ELSE TRUE 
     END
 )
 ORDER BY
     CASE
-        WHEN $7 = FALSE THEN id  
+        WHEN $9 = FALSE THEN id  
         ELSE NULL
     END ASC,
     CASE
-        WHEN $7 = TRUE  THEN id  
+        WHEN $9 = TRUE  THEN id  
         ELSE NULL
     END DESC
-LIMIT $9 OFFSET $8
+LIMIT $11 OFFSET $10
 `
 
 type FilterInvoicesParams struct {
-	AddIndexGet   sql.NullInt32
+	AddIndexGet    sql.NullInt64
-	AddIndexLet   sql.NullInt32
+	AddIndexLet    sql.NullInt64
-	State         sql.NullInt16
+	SettleIndexGet sql.NullInt64
-	CreatedAfter  sql.NullTime
+	SettleIndexLet sql.NullInt64
-	CreatedBefore sql.NullTime
+	State          sql.NullInt16
-	PendingOnly   interface{}
+	CreatedAfter   sql.NullTime
-	Reverse       interface{}
+	CreatedBefore  sql.NullTime
-	NumOffset     int32
+	PendingOnly    interface{}
-	NumLimit      int32
+	Reverse        interface{}
+	NumOffset      int32
+	NumLimit       int32
 }
 
 func (q *Queries) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) ([]Invoice, error) {
 	rows, err := q.db.QueryContext(ctx, filterInvoices,
 		arg.AddIndexGet,
 		arg.AddIndexLet,
+		arg.SettleIndexGet,
+		arg.SettleIndexLet,
 		arg.State,
 		arg.CreatedAfter,
 		arg.CreatedBefore,
@@ -267,12 +137,15 @@ func (q *Queries) FilterInvoices(ctx context.Context, arg FilterInvoicesParams)
 			&i.ID,
 			&i.Hash,
 			&i.Preimage,
+			&i.SettleIndex,
+			&i.SettledAt,
 			&i.Memo,
 			&i.AmountMsat,
 			&i.CltvDelta,
 			&i.Expiry,
 			&i.PaymentAddr,
 			&i.PaymentRequest,
+			&i.PaymentRequestHash,
 			&i.State,
 			&i.AmountPaidMsat,
 			&i.IsAmp,
@@ -295,29 +168,35 @@ func (q *Queries) FilterInvoices(ctx context.Context, arg FilterInvoicesParams)
 
 const getInvoice = `-- name: GetInvoice :many
 
-SELECT id, hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, created_at
+SELECT i.id, i.hash, i.preimage, i.settle_index, i.settled_at, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.payment_request_hash, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at
-FROM invoices
+FROM invoices i
+LEFT JOIN amp_sub_invoices a on i.id = a.invoice_id
 WHERE (
-    id = $1 OR 
+    i.id = $1 OR 
     $1 IS NULL
 ) AND (
-    hash = $2 OR 
+    i.hash = $2 OR 
     $2 IS NULL
 ) AND (
-    preimage = $3 OR 
+    i.preimage = $3 OR 
     $3 IS NULL
 ) AND (
-    payment_addr = $4 OR 
+    i.payment_addr = $4 OR 
     $4 IS NULL
+) AND (
+    a.set_id = $5 OR
+    $5 IS NULL
 )
+GROUP BY i.id
 LIMIT 2
 `
 
 type GetInvoiceParams struct {
-	AddIndex    sql.NullInt32
+	AddIndex    sql.NullInt64
 	Hash        []byte
 	Preimage    []byte
 	PaymentAddr []byte
+	SetID       []byte
 }
 
 // This method may return more than one invoice if filter using multiple fields
@@ -329,6 +208,7 @@ func (q *Queries) GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoi
 		arg.Hash,
 		arg.Preimage,
 		arg.PaymentAddr,
+		arg.SetID,
 	)
 	if err != nil {
 		return nil, err
@@ -341,12 +221,15 @@ func (q *Queries) GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoi
 			&i.ID,
 			&i.Hash,
 			&i.Preimage,
+			&i.SettleIndex,
+			&i.SettledAt,
 			&i.Memo,
 			&i.AmountMsat,
 			&i.CltvDelta,
 			&i.Expiry,
 			&i.PaymentAddr,
 			&i.PaymentRequest,
+			&i.PaymentRequestHash,
 			&i.State,
 			&i.AmountPaidMsat,
 			&i.IsAmp,
@@ -373,7 +256,7 @@ FROM invoice_features
 WHERE invoice_id = $1
 `
 
-func (q *Queries) GetInvoiceFeatures(ctx context.Context, invoiceID int32) ([]InvoiceFeature, error) {
+func (q *Queries) GetInvoiceFeatures(ctx context.Context, invoiceID int64) ([]InvoiceFeature, error) {
 	rows, err := q.db.QueryContext(ctx, getInvoiceFeatures, invoiceID)
 	if err != nil {
 		return nil, err
@@ -408,7 +291,7 @@ type GetInvoiceHTLCCustomRecordsRow struct {
 	Value  []byte
 }
 
-func (q *Queries) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) ([]GetInvoiceHTLCCustomRecordsRow, error) {
+func (q *Queries) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error) {
 	rows, err := q.db.QueryContext(ctx, getInvoiceHTLCCustomRecords, invoiceID)
 	if err != nil {
 		return nil, err
@@ -432,12 +315,12 @@ func (q *Queries) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int
 }
 
 const getInvoiceHTLCs = `-- name: GetInvoiceHTLCs :many
-SELECT id, htlc_id, chan_id, amount_msat, total_mpp_msat, accept_height, accept_time, expiry_height, state, resolve_time, invoice_id
+SELECT id, chan_id, htlc_id, amount_msat, total_mpp_msat, accept_height, accept_time, expiry_height, state, resolve_time, invoice_id
 FROM invoice_htlcs
 WHERE invoice_id = $1
 `
 
-func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]InvoiceHtlc, error) {
+func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int64) ([]InvoiceHtlc, error) {
 	rows, err := q.db.QueryContext(ctx, getInvoiceHTLCs, invoiceID)
 	if err != nil {
 		return nil, err
@@ -448,8 +331,8 @@ func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]Invoi
 		var i InvoiceHtlc
 		if err := rows.Scan(
 			&i.ID,
-			&i.HtlcID,
 			&i.ChanID,
+			&i.HtlcID,
 			&i.AmountMsat,
 			&i.TotalMppMsat,
 			&i.AcceptHeight,
@@ -472,68 +355,35 @@ func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]Invoi
 	return items, nil
 }
 
-const getInvoicePayments = `-- name: GetInvoicePayments :many
-SELECT id, settled_at, amount_paid_msat, invoice_id
-FROM invoice_payments
-WHERE invoice_id = $1
-`
-
-func (q *Queries) GetInvoicePayments(ctx context.Context, invoiceID int32) ([]InvoicePayment, error) {
-	rows, err := q.db.QueryContext(ctx, getInvoicePayments, invoiceID)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []InvoicePayment
-	for rows.Next() {
-		var i InvoicePayment
-		if err := rows.Scan(
-			&i.ID,
-			&i.SettledAt,
-			&i.AmountPaidMsat,
-			&i.InvoiceID,
-		); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Close(); err != nil {
-		return nil, err
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
-}
-
 const insertInvoice = `-- name: InsertInvoice :one
 INSERT INTO invoices (
     hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, 
-    payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, 
+    payment_request, payment_request_hash, state, amount_paid_msat, is_amp,
-    created_at
+    is_hodl, is_keysend, created_at
 ) VALUES (
-    $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14
+    $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15
 ) RETURNING id
 `
 
 type InsertInvoiceParams struct {
-	Hash           []byte
+	Hash               []byte
-	Preimage       []byte
+	Preimage           []byte
-	Memo           sql.NullString
+	Memo               sql.NullString
-	AmountMsat     int64
+	AmountMsat         int64
-	CltvDelta      sql.NullInt32
+	CltvDelta          sql.NullInt32
-	Expiry         int32
+	Expiry             int32
-	PaymentAddr    []byte
+	PaymentAddr        []byte
-	PaymentRequest sql.NullString
+	PaymentRequest     sql.NullString
-	State          int16
+	PaymentRequestHash []byte
-	AmountPaidMsat int64
+	State              int16
-	IsAmp          bool
+	AmountPaidMsat     int64
-	IsHodl         bool
+	IsAmp              bool
-	IsKeysend      bool
+	IsHodl             bool
-	CreatedAt      time.Time
+	IsKeysend          bool
+	CreatedAt          time.Time
 }
 
-func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int32, error) {
+func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int64, error) {
 	row := q.db.QueryRowContext(ctx, insertInvoice,
 		arg.Hash,
 		arg.Preimage,
@@ -543,6 +393,7 @@ func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (i
 		arg.Expiry,
 		arg.PaymentAddr,
 		arg.PaymentRequest,
+		arg.PaymentRequestHash,
 		arg.State,
 		arg.AmountPaidMsat,
 		arg.IsAmp,
@@ -550,7 +401,7 @@ func (q *Queries) InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (i
 		arg.IsKeysend,
 		arg.CreatedAt,
 	)
-	var id int32
+	var id int64
 	err := row.Scan(&id)
 	return id, err
 }
@@ -564,7 +415,7 @@ INSERT INTO invoice_features (
 `
 
 type InsertInvoiceFeatureParams struct {
-	InvoiceID int32
+	InvoiceID int64
 	Feature   int32
 }
 
@@ -573,13 +424,13 @@ func (q *Queries) InsertInvoiceFeature(ctx context.Context, arg InsertInvoiceFea
 	return err
 }
 
-const insertInvoiceHTLC = `-- name: InsertInvoiceHTLC :exec
+const insertInvoiceHTLC = `-- name: InsertInvoiceHTLC :one
 INSERT INTO invoice_htlcs (
     htlc_id, chan_id, amount_msat, total_mpp_msat, accept_height, accept_time,
     expiry_height, state, resolve_time, invoice_id
 ) VALUES (
     $1, $2, $3, $4, $5, $6, $7, $8, $9, $10
-)
+) RETURNING id
 `
 
 type InsertInvoiceHTLCParams struct {
@@ -592,11 +443,11 @@ type InsertInvoiceHTLCParams struct {
 	ExpiryHeight int32
 	State        int16
 	ResolveTime  sql.NullTime
-	InvoiceID    int32
+	InvoiceID    int64
 }
 
-func (q *Queries) InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) error {
+func (q *Queries) InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) (int64, error) {
-	_, err := q.db.ExecContext(ctx, insertInvoiceHTLC,
+	row := q.db.QueryRowContext(ctx, insertInvoiceHTLC,
 		arg.HtlcID,
 		arg.ChanID,
 		arg.AmountMsat,
@@ -608,7 +459,9 @@ func (q *Queries) InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCPa
 		arg.ResolveTime,
 		arg.InvoiceID,
 	)
-	return err
+	var id int64
+	err := row.Scan(&id)
+	return id, err
 }
 
 const insertInvoiceHTLCCustomRecord = `-- name: InsertInvoiceHTLCCustomRecord :exec
@@ -630,64 +483,56 @@ func (q *Queries) InsertInvoiceHTLCCustomRecord(ctx context.Context, arg InsertI
 	return err
 }
 
-const insertInvoicePayment = `-- name: InsertInvoicePayment :one
+const nextInvoiceSettleIndex = `-- name: NextInvoiceSettleIndex :one
-INSERT INTO invoice_payments (
+UPDATE invoice_sequences SET current_value = current_value + 1
-    invoice_id, amount_paid_msat, settled_at 
+WHERE name = 'settle_index'
-) VALUES (
+RETURNING current_value
-    $1, $2, $3
-) RETURNING id
 `
 
-type InsertInvoicePaymentParams struct {
+func (q *Queries) NextInvoiceSettleIndex(ctx context.Context) (int64, error) {
-	InvoiceID      int32
+	row := q.db.QueryRowContext(ctx, nextInvoiceSettleIndex)
-	AmountPaidMsat int64
+	var current_value int64
-	SettledAt      time.Time
+	err := row.Scan(&current_value)
+	return current_value, err
 }
 
-func (q *Queries) InsertInvoicePayment(ctx context.Context, arg InsertInvoicePaymentParams) (int32, error) {
+const updateInvoiceAmountPaid = `-- name: UpdateInvoiceAmountPaid :execresult
-	row := q.db.QueryRowContext(ctx, insertInvoicePayment, arg.InvoiceID, arg.AmountPaidMsat, arg.SettledAt)
+UPDATE invoices
-	var id int32
+SET amount_paid_msat = $2
-	err := row.Scan(&id)
+WHERE id = $1
-	return id, err
-}
-
-const updateInvoice = `-- name: UpdateInvoice :exec
-UPDATE invoices 
-SET preimage=$2, state=$3, amount_paid_msat=$4
-WHERE id=$1
 `
 
-type UpdateInvoiceParams struct {
+type UpdateInvoiceAmountPaidParams struct {
-	ID             int32
+	ID             int64
-	Preimage       []byte
-	State          int16
 	AmountPaidMsat int64
 }
 
-func (q *Queries) UpdateInvoice(ctx context.Context, arg UpdateInvoiceParams) error {
+func (q *Queries) UpdateInvoiceAmountPaid(ctx context.Context, arg UpdateInvoiceAmountPaidParams) (sql.Result, error) {
-	_, err := q.db.ExecContext(ctx, updateInvoice,
+	return q.db.ExecContext(ctx, updateInvoiceAmountPaid, arg.ID, arg.AmountPaidMsat)
-		arg.ID,
-		arg.Preimage,
-		arg.State,
-		arg.AmountPaidMsat,
-	)
-	return err
 }
 
 const updateInvoiceHTLC = `-- name: UpdateInvoiceHTLC :exec
 UPDATE invoice_htlcs 
-SET state=$2, resolve_time=$3
+SET state=$4, resolve_time=$5
-WHERE id = $1
+WHERE htlc_id = $1 AND chan_id = $2 AND invoice_id = $3
 `
 
 type UpdateInvoiceHTLCParams struct {
-	ID          int32
+	HtlcID      int64
+	ChanID      string
+	InvoiceID   int64
 	State       int16
 	ResolveTime sql.NullTime
 }
 
 func (q *Queries) UpdateInvoiceHTLC(ctx context.Context, arg UpdateInvoiceHTLCParams) error {
-	_, err := q.db.ExecContext(ctx, updateInvoiceHTLC, arg.ID, arg.State, arg.ResolveTime)
+	_, err := q.db.ExecContext(ctx, updateInvoiceHTLC,
+		arg.HtlcID,
+		arg.ChanID,
+		arg.InvoiceID,
+		arg.State,
+		arg.ResolveTime,
+	)
 	return err
 }
 
@@ -698,7 +543,7 @@ WHERE invoice_id = $1 AND resolve_time IS NULL
 `
 
 type UpdateInvoiceHTLCsParams struct {
-	InvoiceID   int32
+	InvoiceID   int64
 	State       int16
 	ResolveTime sql.NullTime
 }
@@ -707,3 +552,30 @@ func (q *Queries) UpdateInvoiceHTLCs(ctx context.Context, arg UpdateInvoiceHTLCs
 	_, err := q.db.ExecContext(ctx, updateInvoiceHTLCs, arg.InvoiceID, arg.State, arg.ResolveTime)
 	return err
 }
+
+const updateInvoiceState = `-- name: UpdateInvoiceState :execresult
+UPDATE invoices
+SET state = $2,
+    preimage = COALESCE(preimage, $3),
+    settle_index = COALESCE(settle_index, $4),
+    settled_at = COALESCE(settled_at, $5)
+WHERE id = $1
+`
+
+type UpdateInvoiceStateParams struct {
+	ID          int64
+	State       int16
+	Preimage    []byte
+	SettleIndex sql.NullInt64
+	SettledAt   sql.NullTime
+}
+
+func (q *Queries) UpdateInvoiceState(ctx context.Context, arg UpdateInvoiceStateParams) (sql.Result, error) {
+	return q.db.ExecContext(ctx, updateInvoiceState,
+		arg.ID,
+		arg.State,
+		arg.Preimage,
+		arg.SettleIndex,
+		arg.SettledAt,
+	)
+}

sqldb/sqlc/migrations/000001_invoices.up.sql

@@ -1,7 +1,17 @@
+-- sequences contains all sequences used for invoices.
+CREATE TABLE invoice_sequences (
+    name TEXT PRIMARY KEY,
+    current_value BIGINT NOT NULL
+);
+
+-- Initialize a sequence for the settled index used to track invoice settlement
+-- to remain compatible with the legacy channeldb implementation.
+INSERT INTO invoice_sequences(name, current_value) VALUES ('settle_index', 0);
+
 -- invoices table contains all the information shared by all the invoice types. 
 CREATE TABLE IF NOT EXISTS invoices (
     -- The id of the invoice. Translates to the AddIndex.
-    id INTEGER PRIMARY KEY,
+    id BIGINT PRIMARY KEY,
 
     -- The hash for this invoice. The invoice hash will always identify that 
     -- invoice.
@@ -10,6 +20,14 @@ CREATE TABLE IF NOT EXISTS invoices (
     -- The preimage for the hash in this invoice. Some invoices may have this 
     -- field empty, like unsettled hodl invoices or AMP invoices.
     preimage BLOB,
+    
+    -- If settled, the index is set to the current_value+1 of the settle_index
+    -- seuqence in the invoice_sequences table. If not settled, then it is set
+    -- to NULL.
+    settle_index BIGINT,
+
+    -- When the invoice was settled.
+    settled_at TIMESTAMP,
 
     -- An optional memo to attach along with the invoice. 
     memo TEXT,
@@ -31,7 +49,12 @@ CREATE TABLE IF NOT EXISTS invoices (
 
     -- The encoded payment request for this invoice. Some invoice types may 
     -- not have leave this empty, like Keysends.
-    payment_request TEXT UNIQUE, 
+    payment_request TEXT,
+
+    -- Holds the hash of the payment request. This field is used to ensure that
+    -- there are no duplicates in the database. This trick is needed because
+    -- PostgreSQL has a limitiation of 2712 bytes for unique index rows.
+    payment_request_hash BLOB UNIQUE,
 
     -- The invoice state.
     state SMALLINT NOT NULL,
@@ -58,6 +81,7 @@ CREATE INDEX IF NOT EXISTS invoices_preimage_idx ON invoices(preimage);
 CREATE INDEX IF NOT EXISTS invoices_payment_addr_idx ON invoices(payment_addr);
 CREATE INDEX IF NOT EXISTS invoices_state_idx ON invoices(state);
 CREATE INDEX IF NOT EXISTS invoices_created_at_idx ON invoices(created_at);
+CREATE INDEX IF NOT EXISTS invoices_settled_at_idx ON invoices(settled_at);
 
 -- invoice_features contains the feature bits of an invoice.
 CREATE TABLE IF NOT EXISTS invoice_features (
@@ -65,7 +89,7 @@ CREATE TABLE IF NOT EXISTS invoice_features (
     feature INTEGER NOT NULL,
 
     -- The invoice id this feature belongs to.
-    invoice_id INTEGER NOT NULL REFERENCES invoices(id),
+    invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE,
 
     -- The feature bit is unique per invoice.
     UNIQUE (feature, invoice_id)
@@ -78,15 +102,15 @@ CREATE INDEX IF NOT EXISTS invoice_feature_invoice_id_idx ON invoice_features(in
 CREATE TABLE IF NOT EXISTS invoice_htlcs (
     -- The id for this htlc. Used in foreign keys instead of the 
     -- htlc_id/chan_id combination.
-    id INTEGER PRIMARY KEY,
+    id BIGINT PRIMARY KEY,
+    
+    -- Short chan id indicating the htlc's origin. uint64 stored as text.
+    chan_id TEXT NOT NULL, 
 
     -- The uint64 htlc id. This field is a counter so it is safe to store it as 
     -- int64 in the database. The application layer must check that there is no 
     -- overflow when storing/loading this column.
     htlc_id BIGINT NOT NULL,
-
-    -- Short chan id indicating the htlc's origin. uint64 stored as text.
-    chan_id TEXT NOT NULL, 
     
     -- The htlc's amount in millisatoshis.
     amount_msat BIGINT NOT NULL,
@@ -110,10 +134,10 @@ CREATE TABLE IF NOT EXISTS invoice_htlcs (
     resolve_time TIMESTAMP,
 
     -- The id of the invoice this htlc belongs to.
-    invoice_id INTEGER NOT NULL REFERENCES invoices(id),
+    invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE,
 
     -- The htlc_id and chan_id identify the htlc.
-    UNIQUE (htlc_id, chan_id)
+    UNIQUE (chan_id, htlc_id)
 );
 
 CREATE INDEX IF NOT EXISTS invoice_htlc_invoice_id_idx ON invoice_htlcs(invoice_id);
@@ -129,26 +153,8 @@ CREATE TABLE IF NOT EXISTS invoice_htlc_custom_records (
     value BLOB NOT NULL,
 
     -- The htlc id this record belongs to. 
-    htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id)
+    htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id) ON DELETE CASCADE
 );
 
 CREATE INDEX IF NOT EXISTS invoice_htlc_custom_records_htlc_id_idx ON invoice_htlc_custom_records(htlc_id);
 
--- invoice_payments contains the information of a settled invoice payment. 
-CREATE TABLE IF NOT EXISTS invoice_payments (
-    -- The id for this invoice payment. Translates to SettleIndex.
-    id INTEGER PRIMARY KEY,
-    
-    -- When the payment was settled.
-    settled_at TIMESTAMP NOT NULL,
-
-    -- The amount of the payment in millisatoshis. This is the sum of all the 
-    -- the htlcs settled for this payment.
-    amount_paid_msat BIGINT NOT NULL,
-
-    -- The invoice id this payment is for.
-    invoice_id INTEGER NOT NULL REFERENCES invoices(id)
-);
-
-CREATE INDEX IF NOT EXISTS invoice_payments_settled_at_idx ON invoice_payments(settled_at);
-CREATE INDEX IF NOT EXISTS invoice_payments_invoice_id_idx ON invoice_payments(invoice_id);

sqldb/sqlc/migrations/000002_amp_invoices.up.sql

@@ -1,5 +1,6 @@
--- amp_invoices_payments  
+-- amp_sub_invoices holds all AMP sub-invoices that belong to a given parent
-CREATE TABLE IF NOT EXISTS amp_invoice_payments (
+-- invoice.
+CREATE TABLE IF NOT EXISTS amp_sub_invoices (
     -- The set id identifying the payment. 
     set_id BLOB PRIMARY KEY,
 
@@ -10,26 +11,35 @@ CREATE TABLE IF NOT EXISTS amp_invoice_payments (
     -- Timestamp of when the first htlc for this payment was accepted.
     created_at TIMESTAMP NOT NULL,
 
-    -- If settled, the invoice payment related to this set id.
+    -- When the invoice was settled.
-    settled_index INTEGER REFERENCES invoice_payments(id),
+    settled_at TIMESTAMP,
 
-    -- The invoice id this set id is related to.
+    -- If settled, the index is set to the current_value+1 of the settle_index
-    invoice_id INTEGER NOT NULL REFERENCES invoices(id)
+    -- seuqence in the invoice_sequences table. If not settled, then it is set
+    -- to NULL.
+    settle_index BIGINT,
+
+    -- The id of the parent invoice this AMP invoice belongs to.
+    invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE,
+
+    -- A unique constraint on the set_id and invoice_id is needed to ensure that
+    -- we don't have two AMP invoices for the same invoice.
+    UNIQUE (set_id, invoice_id)
 );
 
-CREATE INDEX IF NOT EXISTS amp_invoice_payments_invoice_id_idx ON amp_invoice_payments(invoice_id);
+CREATE INDEX IF NOT EXISTS amp_sub_invoices_invoice_id_idx ON amp_sub_invoices(invoice_id);
 
 -- amp_invoice_htlcs contains the complementary information for an htlc related 
 -- to an AMP invoice.
-CREATE TABLE IF NOT EXISTS amp_invoice_htlcs (
+CREATE TABLE IF NOT EXISTS amp_sub_invoice_htlcs (
+    -- The invoice id this entry is related to.
+    invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE,
+
     -- The set id identifying the payment this htlc belongs to.
-    set_id BLOB NOT NULL REFERENCES amp_invoice_payments(set_id),
+    set_id BLOB NOT NULL REFERENCES amp_sub_invoices(set_id) ON DELETE CASCADE,
 
     -- The id of the htlc this entry blongs to.
-    htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id),
+    htlc_id BIGINT NOT NULL REFERENCES invoice_htlcs(id) ON DELETE CASCADE,
-
-    -- The invoice id this entry is related to.
-    invoice_id INTEGER NOT NULL REFERENCES invoices(id),
 
     -- The root share for this amp htlc.
     root_share BLOB NOT NULL,
@@ -48,7 +58,7 @@ CREATE TABLE IF NOT EXISTS amp_invoice_htlcs (
     preimage BLOB 
 );
 
-CREATE INDEX IF NOT EXISTS amp_htlcs_set_id_idx ON amp_invoice_htlcs(set_id);
+CREATE INDEX IF NOT EXISTS amp_htlcs_invoice_id_idx ON amp_sub_invoice_htlcs(invoice_id);
-CREATE INDEX IF NOT EXISTS amp_htlcs_invoice_id_idx ON amp_invoice_htlcs(invoice_id);
+CREATE INDEX IF NOT EXISTS amp_htlcs_set_id_idx ON amp_sub_invoice_htlcs(set_id);
-CREATE INDEX IF NOT EXISTS amp_htlcs_htlc_id_idx ON amp_invoice_htlcs(htlc_id);
+CREATE INDEX IF NOT EXISTS amp_htlcs_htlc_id_idx ON amp_sub_invoice_htlcs(htlc_id);
 

sqldb/sqlc/migrations/000003_invoice_events.down.sql

@@ -1,9 +1,6 @@
 DROP TABLE IF EXISTS invoice_events;
-
+DROP INDEX IF EXISTS invoice_events_added_at_idx;
-DROP INDEX IF EXISTS invoice_events_created_at_idx;
-DROP INDEX IF EXISTS invoice_events_invoice_id_idx;
-DROP INDEX IF EXISTS invoice_events_htlc_id_idx;
-DROP INDEX IF EXISTS invoice_events_set_id_idx;
 DROP INDEX IF EXISTS invoice_events_event_type_idx;
-
+DROP INDEX IF EXISTS invoice_events_invoice_id_idx;
+DROP INDEX IF EXISTS invoice_events_set_id_idx;
 DROP TABLE IF EXISTS invoice_event_types;

sqldb/sqlc/migrations/000003_invoice_events.up.sql

@@ -1,59 +1,51 @@
--- invoice_event_types stores the different types of events that can be emitted 
+-- invoice_event_types stores the different types of invoice events. 
--- for invoices.
 CREATE TABLE IF NOT EXISTS invoice_event_types(
     id INTEGER PRIMARY KEY,
 
     description TEXT NOT NULL
 );
 
--- invoice_events stores all events related to the node invoices.
+-- invoice_event_types defines the different types of invoice events.
-CREATE TABLE IF NOT EXISTS invoice_events (
-    id INTEGER PRIMARY KEY,
-
-    -- created_at is the creation time of this event.
-    created_at TIMESTAMP NOT NULL,
-
-    -- invoice_id is the reference to the invoice this event was emitted for.
-    invoice_id INTEGER NOT NULL REFERENCES invoices(id),
-
-    -- htlc_id is the reference to the htlc this event was emitted for, may be 
-    -- null.
-    htlc_id BIGINT REFERENCES invoice_htlcs(htlc_id),
-
-    -- set_id is the reference to the set_id this event was emitted for, may be
-    -- null.
-    set_id BLOB NOT NULL REFERENCES amp_invoice_payments(set_id),
-
-    -- event_type is the type of this event.
-    event_type INTEGER NOT NULL REFERENCES invoice_event_types(id),
-
-    -- event_metadata is a TLV encoding any relevant information for this kind 
-    -- of events.
-    event_metadata BLOB
-);
-
-CREATE INDEX IF NOT EXISTS invoice_events_created_at_idx ON invoice_events(created_at);
-CREATE INDEX IF NOT EXISTS invoice_events_invoice_id_idx ON invoice_events(invoice_id);
-CREATE INDEX IF NOT EXISTS invoice_events_htlc_id_idx ON invoice_events(htlc_id);
-CREATE INDEX IF NOT EXISTS invoice_events_set_id_idx ON invoice_events(set_id);
-CREATE INDEX IF NOT EXISTS invoice_events_event_type_idx ON invoice_events(event_type);
-
-
--- invoice_event_types defines the different types of events that can be emitted
--- for an invoice.
 INSERT INTO invoice_event_types (id, description) 
 VALUES 
-    -- invoice_created is the event emitted when an invoice is created.
+    -- invoice_created is the event type used when an invoice is created.
     (0, 'invoice_created'), 
-    -- invoice_canceled is the event emitted when an invoice is canceled.
+    -- invoice_canceled is the event type used when an invoice is canceled.
-    (1, "invoice_canceled"), 
+    (1, 'invoice_canceled'), 
-    -- invoice_settled is the event emitted when an invoice is settled.
+    -- invoice_settled is the event type used when an invoice is settled.
-    (2, "invoice_settled"),
+    (2, 'invoice_settled'),
-    -- setid_created is the event emitted when the first htlc for the set_id is 
+    -- setid_created is the event type used when an AMP sub invoice
-    -- received. 
+    -- corresponding to the set_id is created.
-    (3, "setid_created"),
+    (3, 'setid_created'),
-    -- setid_canceled is the event emitted when the set_id is canceled.
+    -- setid_canceled is the event type used when an AMP sub invoice
-    (4, "setid_canceled"),
+    -- corresponding to the set_id is canceled.
-    -- setid_settled is the event emitted when the set_id is settled.
+    (4, 'setid_canceled'),
-    (5, "setid_settled");
+    -- setid_settled is the event type used when an AMP sub invoice 
+    -- corresponding to the set_id is settled.
+    (5, 'setid_settled');
 
+
+-- invoice_events stores all major events related to the node's invoices and
+-- AMP sub invoices. This table can be used to create a historical view of what
+-- happened to the node's invoices.
+CREATE TABLE IF NOT EXISTS invoice_events (
+    id BIGINT PRIMARY KEY,
+
+    -- added_at is the timestamp when this event was added.
+    added_at TIMESTAMP NOT NULL,
+
+    -- event_type is the type of this event.
+    event_type INTEGER NOT NULL REFERENCES invoice_event_types(id) ON DELETE CASCADE,
+
+    -- invoice_id is the reference to the invoice this event was added for.
+    invoice_id BIGINT NOT NULL REFERENCES invoices(id) ON DELETE CASCADE,
+
+    -- set_id is the reference to the AMP sub invoice this event was added for.
+    -- May be NULL if the event is not related to an AMP sub invoice.
+    set_id BLOB REFERENCES amp_sub_invoices(set_id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS invoice_events_added_at_idx ON invoice_events(added_at);
+CREATE INDEX IF NOT EXISTS invoice_events_event_type_idx ON invoice_events(event_type);
+CREATE INDEX IF NOT EXISTS invoice_events_invoice_id_idx ON invoice_events(invoice_id);
+CREATE INDEX IF NOT EXISTS invoice_events_set_id_idx ON invoice_events(set_id);

sqldb/sqlc/models.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.18.0
+//   sqlc v1.25.0
 
 package sqlc
 
@@ -9,50 +9,52 @@ import (
 	"time"
 )
 
-type AmpInvoiceHtlc struct {
+type AmpSubInvoice struct {
+	SetID       []byte
+	State       int16
+	CreatedAt   time.Time
+	SettledAt   sql.NullTime
+	SettleIndex sql.NullInt64
+	InvoiceID   int64
+}
+
+type AmpSubInvoiceHtlc struct {
+	InvoiceID  int64
 	SetID      []byte
 	HtlcID     int64
-	InvoiceID  int32
 	RootShare  []byte
 	ChildIndex int64
 	Hash       []byte
 	Preimage   []byte
 }
 
-type AmpInvoicePayment struct {
-	SetID        []byte
-	State        int16
-	CreatedAt    time.Time
-	SettledIndex sql.NullInt32
-	InvoiceID    int32
-}
-
 type Invoice struct {
-	ID             int32
+	ID                 int64
-	Hash           []byte
+	Hash               []byte
-	Preimage       []byte
+	Preimage           []byte
-	Memo           sql.NullString
+	SettleIndex        sql.NullInt64
-	AmountMsat     int64
+	SettledAt          sql.NullTime
-	CltvDelta      sql.NullInt32
+	Memo               sql.NullString
-	Expiry         int32
+	AmountMsat         int64
-	PaymentAddr    []byte
+	CltvDelta          sql.NullInt32
-	PaymentRequest sql.NullString
+	Expiry             int32
-	State          int16
+	PaymentAddr        []byte
-	AmountPaidMsat int64
+	PaymentRequest     sql.NullString
-	IsAmp          bool
+	PaymentRequestHash []byte
-	IsHodl         bool
+	State              int16
-	IsKeysend      bool
+	AmountPaidMsat     int64
-	CreatedAt      time.Time
+	IsAmp              bool
+	IsHodl             bool
+	IsKeysend          bool
+	CreatedAt          time.Time
 }
 
 type InvoiceEvent struct {
-	ID            int32
+	ID        int64
-	CreatedAt     time.Time
+	AddedAt   time.Time
-	InvoiceID     int32
+	EventType int32
-	HtlcID        sql.NullInt64
+	InvoiceID int64
-	SetID         []byte
+	SetID     []byte
-	EventType     int32
-	EventMetadata []byte
 }
 
 type InvoiceEventType struct {
@@ -62,13 +64,13 @@ type InvoiceEventType struct {
 
 type InvoiceFeature struct {
 	Feature   int32
-	InvoiceID int32
+	InvoiceID int64
 }
 
 type InvoiceHtlc struct {
-	ID           int32
+	ID           int64
-	HtlcID       int64
 	ChanID       string
+	HtlcID       int64
 	AmountMsat   int64
 	TotalMppMsat sql.NullInt64
 	AcceptHeight int32
@@ -76,7 +78,7 @@ type InvoiceHtlc struct {
 	ExpiryHeight int32
 	State        int16
 	ResolveTime  sql.NullTime
-	InvoiceID    int32
+	InvoiceID    int64
 }
 
 type InvoiceHtlcCustomRecord struct {
@@ -85,9 +87,7 @@ type InvoiceHtlcCustomRecord struct {
 	HtlcID int64
 }
 
-type InvoicePayment struct {
+type InvoiceSequence struct {
-	ID             int32
+	Name         string
-	SettledAt      time.Time
+	CurrentValue int64
-	AmountPaidMsat int64
-	InvoiceID      int32
 }

sqldb/sqlc/querier.go

@@ -1,51 +1,48 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.18.0
+//   sqlc v1.25.0
 
 package sqlc
 
 import (
 	"context"
+	"database/sql"
 )
 
 type Querier interface {
-	DeleteAMPHTLCCustomRecords(ctx context.Context, invoiceID int32) error
+	DeleteCanceledInvoices(ctx context.Context) (sql.Result, error)
-	DeleteAMPHTLCs(ctx context.Context, invoiceID int32) error
+	DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) (sql.Result, error)
-	DeleteAMPInvoiceHTLC(ctx context.Context, setID []byte) error
+	FetchAMPSubInvoiceHTLCs(ctx context.Context, arg FetchAMPSubInvoiceHTLCsParams) ([]FetchAMPSubInvoiceHTLCsRow, error)
-	DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) error
+	FetchAMPSubInvoices(ctx context.Context, arg FetchAMPSubInvoicesParams) ([]AmpSubInvoice, error)
-	DeleteInvoiceEvents(ctx context.Context, invoiceID int32) error
+	FetchSettledAMPSubInvoices(ctx context.Context, arg FetchSettledAMPSubInvoicesParams) ([]FetchSettledAMPSubInvoicesRow, error)
-	DeleteInvoiceFeatures(ctx context.Context, invoiceID int32) error
-	DeleteInvoiceHTLC(ctx context.Context, htlcID int64) error
-	DeleteInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) error
-	DeleteInvoiceHTLCs(ctx context.Context, invoiceID int32) error
-	FilterInvoicePayments(ctx context.Context, arg FilterInvoicePaymentsParams) ([]FilterInvoicePaymentsRow, error)
 	FilterInvoices(ctx context.Context, arg FilterInvoicesParams) ([]Invoice, error)
-	GetAMPInvoiceHTLCsByInvoiceID(ctx context.Context, invoiceID int32) ([]AmpInvoiceHtlc, error)
+	GetAMPInvoiceID(ctx context.Context, setID []byte) (int64, error)
-	GetAMPInvoiceHTLCsBySetID(ctx context.Context, setID []byte) ([]AmpInvoiceHtlc, error)
 	// This method may return more than one invoice if filter using multiple fields
 	// from different invoices. It is the caller's responsibility to ensure that
 	// we bubble up an error in those cases.
 	GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoice, error)
-	GetInvoiceFeatures(ctx context.Context, invoiceID int32) ([]InvoiceFeature, error)
+	GetInvoiceFeatures(ctx context.Context, invoiceID int64) ([]InvoiceFeature, error)
-	GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int32) ([]GetInvoiceHTLCCustomRecordsRow, error)
+	GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error)
-	GetInvoiceHTLCs(ctx context.Context, invoiceID int32) ([]InvoiceHtlc, error)
+	GetInvoiceHTLCs(ctx context.Context, invoiceID int64) ([]InvoiceHtlc, error)
-	GetInvoicePayments(ctx context.Context, invoiceID int32) ([]InvoicePayment, error)
+	InsertAMPSubInvoiceHTLC(ctx context.Context, arg InsertAMPSubInvoiceHTLCParams) error
-	GetSetIDHTLCsCustomRecords(ctx context.Context, setID []byte) ([]GetSetIDHTLCsCustomRecordsRow, error)
+	InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int64, error)
-	InsertAMPInvoiceHTLC(ctx context.Context, arg InsertAMPInvoiceHTLCParams) error
-	InsertAMPInvoicePayment(ctx context.Context, arg InsertAMPInvoicePaymentParams) error
-	InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int32, error)
-	InsertInvoiceEvent(ctx context.Context, arg InsertInvoiceEventParams) error
 	InsertInvoiceFeature(ctx context.Context, arg InsertInvoiceFeatureParams) error
-	InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) error
+	InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) (int64, error)
 	InsertInvoiceHTLCCustomRecord(ctx context.Context, arg InsertInvoiceHTLCCustomRecordParams) error
-	InsertInvoicePayment(ctx context.Context, arg InsertInvoicePaymentParams) (int32, error)
+	NextInvoiceSettleIndex(ctx context.Context) (int64, error)
-	SelectAMPInvoicePayments(ctx context.Context, arg SelectAMPInvoicePaymentsParams) ([]SelectAMPInvoicePaymentsRow, error)
+	OnAMPSubInvoiceCanceled(ctx context.Context, arg OnAMPSubInvoiceCanceledParams) error
-	SelectInvoiceEvents(ctx context.Context, arg SelectInvoiceEventsParams) ([]InvoiceEvent, error)
+	OnAMPSubInvoiceCreated(ctx context.Context, arg OnAMPSubInvoiceCreatedParams) error
-	UpdateAMPInvoiceHTLC(ctx context.Context, arg UpdateAMPInvoiceHTLCParams) error
+	OnAMPSubInvoiceSettled(ctx context.Context, arg OnAMPSubInvoiceSettledParams) error
-	UpdateAMPPayment(ctx context.Context, arg UpdateAMPPaymentParams) error
+	OnInvoiceCanceled(ctx context.Context, arg OnInvoiceCanceledParams) error
-	UpdateInvoice(ctx context.Context, arg UpdateInvoiceParams) error
+	OnInvoiceCreated(ctx context.Context, arg OnInvoiceCreatedParams) error
+	OnInvoiceSettled(ctx context.Context, arg OnInvoiceSettledParams) error
+	UpdateAMPSubInvoiceHTLCPreimage(ctx context.Context, arg UpdateAMPSubInvoiceHTLCPreimageParams) (sql.Result, error)
+	UpdateAMPSubInvoiceState(ctx context.Context, arg UpdateAMPSubInvoiceStateParams) error
+	UpdateInvoiceAmountPaid(ctx context.Context, arg UpdateInvoiceAmountPaidParams) (sql.Result, error)
 	UpdateInvoiceHTLC(ctx context.Context, arg UpdateInvoiceHTLCParams) error
 	UpdateInvoiceHTLCs(ctx context.Context, arg UpdateInvoiceHTLCsParams) error
+	UpdateInvoiceState(ctx context.Context, arg UpdateInvoiceStateParams) (sql.Result, error)
+	UpsertAMPSubInvoice(ctx context.Context, arg UpsertAMPSubInvoiceParams) (sql.Result, error)
 }
 
 var _ Querier = (*Queries)(nil)

sqldb/sqlc/queries/amp_invoices.sql

@@ -1,79 +1,67 @@
--- name: InsertAMPInvoicePayment :exec
+-- name: UpsertAMPSubInvoice :execresult
-INSERT INTO amp_invoice_payments (
+INSERT INTO amp_sub_invoices (
-    set_id, state, created_at, settled_index, invoice_id
+    set_id, state, created_at, invoice_id
 ) VALUES (
-    $1, $2, $3, $4, $5
+    $1, $2, $3, $4
-); 
+) ON CONFLICT (set_id, invoice_id) DO NOTHING;
 
--- name: SelectAMPInvoicePayments :many
+-- name: GetAMPInvoiceID :one
-SELECT aip.*, ip.*
+SELECT invoice_id FROM amp_sub_invoices WHERE set_id = $1;
-FROM amp_invoice_payments aip LEFT JOIN invoice_payments ip ON aip.settled_index = ip.id
-WHERE (
-    set_id = sqlc.narg('set_id') OR 
-    sqlc.narg('set_id') IS NULL
-) AND (
-    aip.settled_index = sqlc.narg('settled_index') OR 
-    sqlc.narg('settled_index') IS NULL
-) AND (
-    aip.invoice_id = sqlc.narg('invoice_id') OR 
-    sqlc.narg('invoice_id') IS NULL
-); 
 
--- name: UpdateAMPPayment :exec
+-- name: UpdateAMPSubInvoiceState :exec
-UPDATE amp_invoice_payments
+UPDATE amp_sub_invoices
-SET state = $1, settled_index = $2
+SET state = $2, 
-WHERE state = 0 AND (
+    settle_index = COALESCE(settle_index, $3),
-    set_id = sqlc.narg('set_id') OR 
+    settled_at = COALESCE(settled_at, $4)
-    sqlc.narg('set_id') IS NULL
+WHERE set_id = $1; 
-) AND (
-    invoice_id = sqlc.narg('invoice_id') OR 
-    sqlc.narg('invoice_id') IS NULL
-); 
 
--- name: InsertAMPInvoiceHTLC :exec
+-- name: InsertAMPSubInvoiceHTLC :exec
-INSERT INTO amp_invoice_htlcs (
+INSERT INTO amp_sub_invoice_htlcs (
-    set_id, htlc_id, root_share, child_index, hash, preimage
+    invoice_id, set_id, htlc_id, root_share, child_index, hash, preimage
 ) VALUES (
-    $1, $2, $3, $4, $5, $6
+    $1, $2, $3, $4, $5, $6, $7
 );
 
--- name: GetAMPInvoiceHTLCsBySetID :many
+-- name: FetchAMPSubInvoices :many
 SELECT *
-FROM amp_invoice_htlcs
+FROM amp_sub_invoices
-WHERE set_id = $1;
+WHERE invoice_id = $1 
+AND (
+    set_id = sqlc.narg('set_id') OR 
+    sqlc.narg('set_id') IS NULL
+);
 
--- name: GetAMPInvoiceHTLCsByInvoiceID :many
+-- name: FetchAMPSubInvoiceHTLCs :many
-SELECT *
+SELECT 
-FROM amp_invoice_htlcs
+    amp.set_id, amp.root_share, amp.child_index, amp.hash, amp.preimage, 
-WHERE invoice_id = $1;
+    invoice_htlcs.*
+FROM amp_sub_invoice_htlcs amp
+INNER JOIN invoice_htlcs ON amp.htlc_id = invoice_htlcs.id
+WHERE amp.invoice_id = $1
+AND (
+    set_id = sqlc.narg('set_id') OR 
+    sqlc.narg('set_id') IS NULL
+);
 
--- name: GetSetIDHTLCsCustomRecords :many
+-- name: FetchSettledAMPSubInvoices :many
-SELECT ihcr.htlc_id, key, value
+SELECT 
-FROM amp_invoice_htlcs aih JOIN invoice_htlc_custom_records ihcr ON aih.id=ihcr.htlc_id 
+    a.set_id, 
-WHERE aih.set_id = $1;
+    a.settle_index as amp_settle_index, 
-
+    a.settled_at as amp_settled_at,
--- name: UpdateAMPInvoiceHTLC :exec
+    i.*
-UPDATE amp_invoice_htlcs
+FROM amp_sub_invoices a
-SET preimage = $1
+INNER JOIN invoices i ON a.invoice_id = i.id
-WHERE htlc_id = $2;
+WHERE (
-
+    a.settle_index >= sqlc.narg('settle_index_get') OR
--- name: DeleteAMPHTLCCustomRecords :exec
+    sqlc.narg('settle_index_get') IS NULL
-WITH htlc_ids AS (
+) AND (
-    SELECT htlc_id
+    a.settle_index <= sqlc.narg('settle_index_let') OR
-    FROM amp_invoice_htlcs 
+    sqlc.narg('settle_index_let') IS NULL
-    WHERE invoice_id = $1
+);
-)
-DELETE
-FROM invoice_htlc_custom_records
-WHERE htlc_id IN (SELECT id FROM htlc_ids);
-
--- name: DeleteAMPHTLCs :exec
-DELETE 
-FROM amp_invoice_htlcs  
-WHERE invoice_id = $1;
-
--- name: DeleteAMPInvoiceHTLC :exec
-DELETE 
-FROM amp_invoice_htlcs
-WHERE set_id = $1;
 
+-- name: UpdateAMPSubInvoiceHTLCPreimage :execresult
+UPDATE amp_sub_invoice_htlcs AS a
+SET preimage = $4
+WHERE a.invoice_id = $1 AND a.set_id = $2 AND a.htlc_id = (
+    SELECT id FROM invoice_htlcs AS i WHERE i.htlc_id = $3
+);

sqldb/sqlc/queries/invoice_events.sql

@@ -1,35 +1,41 @@
--- name: InsertInvoiceEvent :exec
+-- name: OnInvoiceCreated :exec
 INSERT INTO invoice_events (
-    created_at, invoice_id, htlc_id, set_id, event_type, event_metadata
+    added_at, event_type, invoice_id
 ) VALUES (
-    $1, $2, $3, $4, $5, $6
+    $1, 0, $2
 );
 
--- name: SelectInvoiceEvents :many
+-- name: OnInvoiceCanceled :exec
-SELECT *
+INSERT INTO invoice_events (
-FROM invoice_events
+    added_at, event_type, invoice_id
-WHERE (
+) VALUES (
-    invoice_id = sqlc.narg('invoice_id') OR 
+    $1, 1, $2
-    sqlc.narg('invoice_id') IS NULL
+);
-) AND (
-    htlc_id = sqlc.narg('htlc_id') OR 
-    sqlc.narg('htlc_id') IS NULL
-) AND (
-    set_id = sqlc.narg('set_id') OR 
-    sqlc.narg('set_id') IS NULL
-) AND (
-    event_type = sqlc.narg('event_type') OR 
-    sqlc.narg('event_type') IS NULL
-) AND (
-    created_at >= sqlc.narg('created_after') OR 
-    sqlc.narg('created_after') IS NULL
-) AND (
-    created_at <= sqlc.narg('created_before') OR 
-    sqlc.narg('created_before') IS NULL
-) 
-LIMIT @num_limit OFFSET @num_offset;
 
--- name: DeleteInvoiceEvents :exec
+-- name: OnInvoiceSettled :exec
-DELETE
+INSERT INTO invoice_events (
-FROM invoice_events
+    added_at, event_type, invoice_id
-WHERE invoice_id = $1;
+) VALUES (
+    $1, 2, $2
+);
+
+-- name: OnAMPSubInvoiceCreated :exec
+INSERT INTO invoice_events (
+    added_at, event_type, invoice_id, set_id
+) VALUES (
+    $1, 3, $2, $3
+);
+
+-- name: OnAMPSubInvoiceCanceled :exec
+INSERT INTO invoice_events (
+    added_at, event_type, invoice_id, set_id
+) VALUES (
+    $1, 4, $2, $3
+);
+
+-- name: OnAMPSubInvoiceSettled :exec
+INSERT INTO invoice_events (
+    added_at, event_type, invoice_id, set_id
+) VALUES (
+    $1, 5, $2, $3
+);

sqldb/sqlc/queries/invoices.sql

@@ -1,10 +1,10 @@
 -- name: InsertInvoice :one
 INSERT INTO invoices (
     hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, 
-    payment_request, state, amount_paid_msat, is_amp, is_hodl, is_keysend, 
+    payment_request, payment_request_hash, state, amount_paid_msat, is_amp,
-    created_at
+    is_hodl, is_keysend, created_at
 ) VALUES (
-    $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14
+    $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15
 ) RETURNING id;
 
 -- name: InsertInvoiceFeature :exec
@@ -19,35 +19,36 @@ SELECT *
 FROM invoice_features
 WHERE invoice_id = $1;
 
--- name: DeleteInvoiceFeatures :exec
-DELETE 
-FROM invoice_features
-WHERE invoice_id = $1;
-
 -- This method may return more than one invoice if filter using multiple fields
 -- from different invoices. It is the caller's responsibility to ensure that 
 -- we bubble up an error in those cases.
 
 -- name: GetInvoice :many
-SELECT *
+SELECT i.*
-FROM invoices
+FROM invoices i
+LEFT JOIN amp_sub_invoices a on i.id = a.invoice_id
 WHERE (
-    id = sqlc.narg('add_index') OR 
+    i.id = sqlc.narg('add_index') OR 
     sqlc.narg('add_index') IS NULL
 ) AND (
-    hash = sqlc.narg('hash') OR 
+    i.hash = sqlc.narg('hash') OR 
     sqlc.narg('hash') IS NULL
 ) AND (
-    preimage = sqlc.narg('preimage') OR 
+    i.preimage = sqlc.narg('preimage') OR 
     sqlc.narg('preimage') IS NULL
 ) AND (
-    payment_addr = sqlc.narg('payment_addr') OR 
+    i.payment_addr = sqlc.narg('payment_addr') OR 
     sqlc.narg('payment_addr') IS NULL
+) AND (
+    a.set_id = sqlc.narg('set_id') OR
+    sqlc.narg('set_id') IS NULL
 )
+GROUP BY i.id
 LIMIT 2;
 
 -- name: FilterInvoices :many
-SELECT * 
+SELECT
+    invoices.*
 FROM invoices
 WHERE (
     id >= sqlc.narg('add_index_get') OR 
@@ -55,6 +56,12 @@ WHERE (
 ) AND (
     id <= sqlc.narg('add_index_let') OR 
     sqlc.narg('add_index_let') IS NULL
+) AND (
+    settle_index >= sqlc.narg('settle_index_get') OR
+    sqlc.narg('settle_index_get') IS NULL
+) AND (
+    settle_index <= sqlc.narg('settle_index_let') OR
+    sqlc.narg('settle_index_let') IS NULL
 ) AND (
     state = sqlc.narg('state') OR 
     sqlc.narg('state') IS NULL
@@ -81,12 +88,25 @@ ORDER BY
     END DESC
 LIMIT @num_limit OFFSET @num_offset;
 
--- name: UpdateInvoice :exec 
+-- name: UpdateInvoiceState :execresult
-UPDATE invoices 
+UPDATE invoices
-SET preimage=$2, state=$3, amount_paid_msat=$4
+SET state = $2,
-WHERE id=$1;
+    preimage = COALESCE(preimage, $3),
+    settle_index = COALESCE(settle_index, $4),
+    settled_at = COALESCE(settled_at, $5)
+WHERE id = $1;
 
--- name: DeleteInvoice :exec
+-- name: UpdateInvoiceAmountPaid :execresult
+UPDATE invoices
+SET amount_paid_msat = $2
+WHERE id = $1;
+
+-- name: NextInvoiceSettleIndex :one
+UPDATE invoice_sequences SET current_value = current_value + 1
+WHERE name = 'settle_index'
+RETURNING current_value;
+
+-- name: DeleteInvoice :execresult
 DELETE 
 FROM invoices 
 WHERE (
@@ -96,20 +116,25 @@ WHERE (
     hash = sqlc.narg('hash') OR 
     sqlc.narg('hash') IS NULL
 ) AND (
-    preimage = sqlc.narg('preimage') OR 
+    settle_index = sqlc.narg('settle_index') OR 
-    sqlc.narg('preimage') IS NULL
+    sqlc.narg('settle_index') IS NULL
 ) AND (
     payment_addr = sqlc.narg('payment_addr') OR
     sqlc.narg('payment_addr') IS NULL
 );
 
--- name: InsertInvoiceHTLC :exec
+-- name: DeleteCanceledInvoices :execresult
+DELETE
+FROM invoices
+WHERE state = 2;
+
+-- name: InsertInvoiceHTLC :one
 INSERT INTO invoice_htlcs (
     htlc_id, chan_id, amount_msat, total_mpp_msat, accept_height, accept_time,
     expiry_height, state, resolve_time, invoice_id
 ) VALUES (
     $1, $2, $3, $4, $5, $6, $7, $8, $9, $10
-);
+) RETURNING id;
 
 -- name: GetInvoiceHTLCs :many
 SELECT *
@@ -118,25 +143,14 @@ WHERE invoice_id = $1;
 
 -- name: UpdateInvoiceHTLC :exec
 UPDATE invoice_htlcs 
-SET state=$2, resolve_time=$3
+SET state=$4, resolve_time=$5
-WHERE id = $1;
+WHERE htlc_id = $1 AND chan_id = $2 AND invoice_id = $3;
 
 -- name: UpdateInvoiceHTLCs :exec
 UPDATE invoice_htlcs 
 SET state=$2, resolve_time=$3
 WHERE invoice_id = $1 AND resolve_time IS NULL;
 
-
--- name: DeleteInvoiceHTLC :exec
-DELETE
-FROM invoice_htlcs
-WHERE htlc_id = $1;
-
--- name: DeleteInvoiceHTLCs :exec
-DELETE
-FROM invoice_htlcs
-WHERE invoice_id = $1;
-
 -- name: InsertInvoiceHTLCCustomRecord :exec
 INSERT INTO invoice_htlc_custom_records (
     key, value, htlc_id 
@@ -148,49 +162,3 @@ INSERT INTO invoice_htlc_custom_records (
 SELECT ihcr.htlc_id, key, value
 FROM invoice_htlcs ih JOIN invoice_htlc_custom_records ihcr ON ih.id=ihcr.htlc_id 
 WHERE ih.invoice_id = $1;
-
--- name: DeleteInvoiceHTLCCustomRecords :exec
-WITH htlc_ids AS (
-    SELECT ih.id
-    FROM invoice_htlcs ih JOIN invoice_htlc_custom_records ihcr ON ih.id=ihcr.htlc_id 
-    WHERE ih.invoice_id = $1
-)
-DELETE
-FROM invoice_htlc_custom_records
-WHERE htlc_id IN (SELECT id FROM htlc_ids);
-
--- name: InsertInvoicePayment :one
-INSERT INTO invoice_payments (
-    invoice_id, amount_paid_msat, settled_at 
-) VALUES (
-    $1, $2, $3
-) RETURNING id;
-
--- name: GetInvoicePayments :many
-SELECT *
-FROM invoice_payments
-WHERE invoice_id = $1;
-
--- name: FilterInvoicePayments :many
-SELECT  
-    ip.id AS settle_index, ip.amount_paid_msat, ip.settled_at AS settle_date, 
-    i.* 
-FROM invoice_payments ip JOIN invoices i ON ip.invoice_id = i.id
-WHERE (
-    ip.id >= sqlc.narg('settle_index_get') OR 
-    sqlc.narg('settle_index_get') IS NULL
-) AND (
-    ip.settled_at >= sqlc.narg('settled_after') OR 
-    sqlc.narg('settled_after') IS NULL
-)
-ORDER BY
-    CASE
-        WHEN sqlc.narg('reverse') = FALSE THEN ip.id  
-        ELSE NULL
-    END ASC,
-    CASE
-        WHEN sqlc.narg('reverse') = TRUE THEN ip.id  
-        ELSE NULL
-    END DESC
-LIMIT @num_limit OFFSET @num_offset;
-

sqldb/sqlerrors.go

@@ -1,3 +1,5 @@
+//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64))
+
 package sqldb
 
 import (
@@ -46,6 +48,11 @@ func parseSqliteError(sqliteErr *sqlite.Error) error {
 			DBError: sqliteErr,
 		}
 
+	case sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY:
+		return &ErrSQLUniqueConstraintViolation{
+			DBError: sqliteErr,
+		}
+
 	// Database is currently busy, so we'll need to try again.
 	case sqlite3.SQLITE_BUSY:
 		return &ErrSerializationError{

sqldb/sqlerrors_no_sqlite.go

@@ -0,0 +1,86 @@
+//go:build js || (windows && (arm || 386)) || (linux && (ppc64 || mips || mipsle || mips64))
+
+package sqldb
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/jackc/pgconn"
+	"github.com/jackc/pgerrcode"
+)
+
+var (
+	// ErrRetriesExceeded is returned when a transaction is retried more
+	// than the max allowed valued without a success.
+	ErrRetriesExceeded = errors.New("db tx retries exceeded")
+)
+
+// MapSQLError attempts to interpret a given error as a database agnostic SQL
+// error.
+func MapSQLError(err error) error {
+	// Attempt to interpret the error as a postgres error.
+	var pqErr *pgconn.PgError
+	if errors.As(err, &pqErr) {
+		return parsePostgresError(pqErr)
+	}
+
+	// Return original error if it could not be classified as a database
+	// specific error.
+	return err
+}
+
+// parsePostgresError attempts to parse a postgres error as a database agnostic
+// SQL error.
+func parsePostgresError(pqErr *pgconn.PgError) error {
+	switch pqErr.Code {
+	// Handle unique constraint violation error.
+	case pgerrcode.UniqueViolation:
+		return &ErrSQLUniqueConstraintViolation{
+			DBError: pqErr,
+		}
+
+	// Unable to serialize the transaction, so we'll need to try again.
+	case pgerrcode.SerializationFailure:
+		return &ErrSerializationError{
+			DBError: pqErr,
+		}
+
+	default:
+		return fmt.Errorf("unknown postgres error: %w", pqErr)
+	}
+}
+
+// ErrSQLUniqueConstraintViolation is an error type which represents a database
+// agnostic SQL unique constraint violation.
+type ErrSQLUniqueConstraintViolation struct {
+	DBError error
+}
+
+func (e ErrSQLUniqueConstraintViolation) Error() string {
+	return fmt.Sprintf("sql unique constraint violation: %v", e.DBError)
+}
+
+// ErrSerializationError is an error type which represents a database agnostic
+// error that a transaction couldn't be serialized with other concurrent db
+// transactions.
+type ErrSerializationError struct {
+	DBError error
+}
+
+// Unwrap returns the wrapped error.
+func (e ErrSerializationError) Unwrap() error {
+	return e.DBError
+}
+
+// Error returns the error message.
+func (e ErrSerializationError) Error() string {
+	return e.DBError.Error()
+}
+
+// IsSerializationError returns true if the given error is a serialization
+// error.
+func IsSerializationError(err error) bool {
+	var serializationError *ErrSerializationError
+	return errors.As(err, &serializationError)
+}

sqldb/sqlite.go

@@ -1,3 +1,5 @@
+//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64))
+
 package sqldb
 
 import (
@@ -6,7 +8,6 @@ import (
 	"net/url"
 	"path/filepath"
 	"testing"
-	"time"
 
 	sqlite_migrate "github.com/golang-migrate/migrate/v4/database/sqlite"
 	"github.com/lightningnetwork/lnd/sqldb/sqlc"
@@ -23,32 +24,8 @@ const (
 	// sqliteTxLockImmediate is a dsn option used to ensure that write
 	// transactions are started immediately.
 	sqliteTxLockImmediate = "_txlock=immediate"
-
-	// defaultMaxConns is the number of permitted active and idle
-	// connections. We want to limit this so it isn't unlimited. We use the
-	// same value for the number of idle connections as, this can speed up
-	// queries given a new connection doesn't need to be established each
-	// time.
-	defaultMaxConns = 25
-
-	// connIdleLifetime is the amount of time a connection can be idle.
-	connIdleLifetime = 5 * time.Minute
 )
 
-// SqliteConfig holds all the config arguments needed to interact with our
-// sqlite DB.
-//
-//nolint:lll
-type SqliteConfig struct {
-	// SkipMigrations if true, then all the tables will be created on start
-	// up if they don't already exist.
-	SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."`
-
-	// DatabaseFileName is the full file path where the database file can be
-	// found.
-	DatabaseFileName string `long:"dbfile" description:"The full path to the database."`
-}
-
 // SqliteStore is a database store implementation that uses a sqlite backend.
 type SqliteStore struct {
 	cfg *SqliteConfig
@@ -58,7 +35,7 @@ type SqliteStore struct {
 
 // NewSqliteStore attempts to open a new sqlite database based on the passed
 // config.
-func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) {
+func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) {
 	// The set of pragma options are accepted using query options. For now
 	// we only want to ensure that foreign key constraints are properly
 	// enforced.
@@ -107,7 +84,7 @@ func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) {
 	// details on the formatting here, see the modernc.org/sqlite docs:
 	// https://pkg.go.dev/modernc.org/sqlite#Driver.Open.
 	dsn := fmt.Sprintf(
-		"%v?%v&%v", cfg.DatabaseFileName, sqliteOptions.Encode(),
+		"%v?%v&%v", dbPath, sqliteOptions.Encode(),
 		sqliteTxLockImmediate,
 	)
 	db, err := sql.Open("sqlite", dsn)
@@ -133,8 +110,16 @@ func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) {
 			return nil, err
 		}
 
+		// We use INTEGER PRIMARY KEY for sqlite, because it acts as a
+		// ROWID alias which is 8 bytes big and also autoincrements.
+		// It's important to use the ROWID as a primary key because the
+		// key look ups are almost twice as fast
+		sqliteFS := newReplacerFS(sqlSchemas, map[string]string{
+			"BIGINT PRIMARY KEY": "INTEGER PRIMARY KEY",
+		})
+
 		err = applyMigrations(
-			sqlSchemas, driver, "sqlc/migrations", "sqlc",
+			sqliteFS, driver, "sqlc/migrations", "sqlc",
 		)
 		if err != nil {
 			return nil, err
@@ -163,9 +148,8 @@ func NewTestSqliteDB(t *testing.T) *SqliteStore {
 	// an in mem version to speed up tests
 	dbFileName := filepath.Join(t.TempDir(), "tmp.db")
 	sqlDB, err := NewSqliteStore(&SqliteConfig{
-		DatabaseFileName: dbFileName,
+		SkipMigrations: false,
-		SkipMigrations:   false,
+	}, dbFileName)
-	})
 	require.NoError(t, err)
 
 	t.Cleanup(func() {

sqldb/sqlutils.go

@@ -0,0 +1,54 @@
+package sqldb
+
+import (
+	"database/sql"
+	"time"
+
+	"golang.org/x/exp/constraints"
+)
+
+// sqlInt32 turns a numerical integer type into the NullInt32 that sql/sqlc
+// uses when an integer field can be permitted to be NULL.
+//
+// We use this constraints.Integer constraint here which maps to all signed and
+// unsigned integer types.
+func sqlInt32[T constraints.Integer](num T) sql.NullInt32 {
+	return sql.NullInt32{
+		Int32: int32(num),
+		Valid: true,
+	}
+}
+
+// sqlInt64 turns a numerical integer type into the NullInt64 that sql/sqlc
+// uses when an integer field can be permitted to be NULL.
+//
+// We use this constraints.Integer constraint here which maps to all signed and
+// unsigned integer types.
+func sqlInt64[T constraints.Integer](num T) sql.NullInt64 {
+	return sql.NullInt64{
+		Int64: int64(num),
+		Valid: true,
+	}
+}
+
+// sqlStr turns a string into the NullString that sql/sqlc uses when a string
+// can be permitted to be NULL.
+func sqlStr(s string) sql.NullString {
+	if s == "" {
+		return sql.NullString{}
+	}
+
+	return sql.NullString{
+		String: s,
+		Valid:  true,
+	}
+}
+
+// sqlTime turns a time.Time into the NullTime that sql/sqlc uses when a time
+// can be permitted to be NULL.
+func sqlTime(t time.Time) sql.NullTime {
+	return sql.NullTime{
+		Time:  t,
+		Valid: true,
+	}
+}