From 57fd965932af88ad9389fdf68bdcc41d71d41cb7 Mon Sep 17 00:00:00 2001 From: ziggie Date: Wed, 11 Feb 2026 12:40:05 +0100 Subject: [PATCH 1/5] sqldb: wire up BusyTimeout config for SQLite store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SqliteConfig.BusyTimeout field existed but was never used — the busy_timeout pragma was hardcoded to 5000ms. Add a busyTimeoutMs() helper that returns the configured value or falls back to the 5000ms default, and use it when constructing the SQLite DSN. --- sqldb/config.go | 16 ++++++++++++++++ sqldb/sqlite.go | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/sqldb/config.go b/sqldb/config.go index 59801dbea..ebf386f54 100644 --- a/sqldb/config.go +++ b/sqldb/config.go @@ -31,6 +31,22 @@ type SqliteConfig struct { QueryConfig `group:"query" namespace:"query"` } +const ( + // defaultBusyTimeoutMs is the default busy_timeout value in + // milliseconds, used when no BusyTimeout is configured. + defaultBusyTimeoutMs = 5000 +) + +// busyTimeoutMs returns the busy_timeout value in milliseconds. If +// BusyTimeout is not set, it returns the default value. +func (s *SqliteConfig) busyTimeoutMs() int64 { + if s.BusyTimeout > 0 { + return s.BusyTimeout.Milliseconds() + } + + return defaultBusyTimeoutMs +} + // Validate checks that the SqliteConfig values are valid. func (p *SqliteConfig) Validate() error { if err := p.QueryConfig.Validate(true); err != nil { diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index 2b2f7be13..fc724e939 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -70,7 +70,7 @@ func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { }, { name: "busy_timeout", - value: "5000", + value: fmt.Sprintf("%d", cfg.busyTimeoutMs()), }, { // With the WAL mode, this ensures that we also do an From a7d1a30cf25e23571b728bb426e7fbedd6d471fa Mon Sep 17 00:00:00 2001 From: ziggie Date: Wed, 11 Feb 2026 12:40:25 +0100 Subject: [PATCH 2/5] sqldb: wire up PragmaOptions config for SQLite store The SqliteConfig.PragmaOptions field existed but was never appended to the DSN. Add the loop to apply user-specified pragma options after the built-in ones, matching the existing behavior in kvdb/sqlite. --- sqldb/sqlite.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index fc724e939..ede2999ad 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -100,6 +100,12 @@ func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { ) } + // Then we add any user specified pragma options. Note that these can + // be of the form: "key=value", "key(N)" or "key". + for _, option := range cfg.PragmaOptions { + sqliteOptions.Add(sqliteOptionPrefix, option) + } + // Construct the DSN which is just the database file name, appended // with the series of pragma options as a query URL string. For more // details on the formatting here, see the modernc.org/sqlite docs: From d87d93b11490042afca7ae4abf902d048144daad Mon Sep 17 00:00:00 2001 From: ziggie Date: Wed, 11 Feb 2026 13:01:53 +0100 Subject: [PATCH 3/5] sqldb+lncfg: use SQLite-appropriate default for max connections Both the sqldb and kvdb SQLite layers were not using a sensible default for MaxConnections. The sqldb store used defaultMaxConns (25) which is meant for Postgres, and the kvdb path passed 0 (unlimited) to sqlbase.Init when unconfigured. Add a MaxConns() method on SqliteConfig that returns the configured value or defaults to 2, appropriate for SQLite's single-writer model. Use it in both sqldb/sqlite.go and lncfg/db.go so both layers share the same default. --- lncfg/db.go | 2 +- sqldb/config.go | 18 ++++++++++++++++++ sqldb/sqlite.go | 4 ++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lncfg/db.go b/lncfg/db.go index 5bd4d2e19..2dec1bd63 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -207,7 +207,7 @@ func (db *DB) Init(ctx context.Context, dbPath string) error { sqlbase.Init(db.Postgres.MaxConnections) case db.Backend == SqliteBackend: - sqlbase.Init(db.Sqlite.MaxConnections) + sqlbase.Init(db.Sqlite.MaxConns()) } return nil diff --git a/sqldb/config.go b/sqldb/config.go index ebf386f54..a5e8122ab 100644 --- a/sqldb/config.go +++ b/sqldb/config.go @@ -32,6 +32,12 @@ type SqliteConfig struct { } const ( + // defaultSqliteMaxConns is the default number of maximum open + // connections for SQLite. SQLite only supports a single writer, so a + // low default reduces contention on the busy_timeout and limits + // resource usage, especially on mobile. + defaultSqliteMaxConns = 2 + // defaultBusyTimeoutMs is the default busy_timeout value in // milliseconds, used when no BusyTimeout is configured. defaultBusyTimeoutMs = 5000 @@ -47,6 +53,18 @@ func (s *SqliteConfig) busyTimeoutMs() int64 { return defaultBusyTimeoutMs } +// MaxConns returns the effective maximum number of open connections. If +// MaxConnections is not set, it returns a default of 2. This low default is +// chosen because SQLite only supports a single writer, which helps reduce +// contention and resource usage. +func (s *SqliteConfig) MaxConns() int { + if s.MaxConnections > 0 { + return s.MaxConnections + } + + return defaultSqliteMaxConns +} + // Validate checks that the SqliteConfig values are valid. func (p *SqliteConfig) Validate() error { if err := p.QueryConfig.Validate(true); err != nil { diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index ede2999ad..1ed26810d 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -136,8 +136,8 @@ func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { err) } - db.SetMaxOpenConns(defaultMaxConns) - db.SetMaxIdleConns(defaultMaxConns) + db.SetMaxOpenConns(cfg.MaxConns()) + db.SetMaxIdleConns(cfg.MaxConns()) db.SetConnMaxLifetime(connIdleLifetime) queries := sqlc.New(db) From 325f83c179e22be57793042e00a33ea0268b9583 Mon Sep 17 00:00:00 2001 From: ziggie Date: Wed, 11 Feb 2026 13:10:28 +0100 Subject: [PATCH 4/5] sqldb+lncfg: consolidate SQLite default constants Move the SQLite default constants (max connections, busy timeout) to sqldb/config.go as the single source of truth and export them. Remove the duplicate definitions from lncfg/db.go and reference the sqldb constants instead. --- lncfg/db.go | 7 ++----- sqldb/config.go | 14 +++++++------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lncfg/db.go b/lncfg/db.go index 2dec1bd63..6835382e2 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -40,9 +40,6 @@ const ( DefaultBatchCommitInterval = 500 * time.Millisecond defaultPostgresMaxConnections = 50 - defaultSqliteMaxConnections = 2 - - defaultSqliteBusyTimeout = 5 * time.Second // NSChannelDB is the namespace name that we use for the combined graph // and channel state DB. @@ -126,8 +123,8 @@ func DefaultDB() *DB { QueryConfig: *sqldb.DefaultPostgresConfig(), }, Sqlite: &sqldb.SqliteConfig{ - MaxConnections: defaultSqliteMaxConnections, - BusyTimeout: defaultSqliteBusyTimeout, + MaxConnections: sqldb.DefaultSqliteMaxConns, + BusyTimeout: sqldb.DefaultSqliteBusyTimeout, QueryConfig: *sqldb.DefaultSQLiteConfig(), }, UseNativeSQL: false, diff --git a/sqldb/config.go b/sqldb/config.go index a5e8122ab..bf2a8deb8 100644 --- a/sqldb/config.go +++ b/sqldb/config.go @@ -32,15 +32,15 @@ type SqliteConfig struct { } const ( - // defaultSqliteMaxConns is the default number of maximum open + // DefaultSqliteMaxConns is the default number of maximum open // connections for SQLite. SQLite only supports a single writer, so a // low default reduces contention on the busy_timeout and limits // resource usage, especially on mobile. - defaultSqliteMaxConns = 2 + DefaultSqliteMaxConns = 2 - // defaultBusyTimeoutMs is the default busy_timeout value in - // milliseconds, used when no BusyTimeout is configured. - defaultBusyTimeoutMs = 5000 + // DefaultSqliteBusyTimeout is the default busy_timeout value used + // when no BusyTimeout is configured. + DefaultSqliteBusyTimeout = 5 * time.Second ) // busyTimeoutMs returns the busy_timeout value in milliseconds. If @@ -50,7 +50,7 @@ func (s *SqliteConfig) busyTimeoutMs() int64 { return s.BusyTimeout.Milliseconds() } - return defaultBusyTimeoutMs + return DefaultSqliteBusyTimeout.Milliseconds() } // MaxConns returns the effective maximum number of open connections. If @@ -62,7 +62,7 @@ func (s *SqliteConfig) MaxConns() int { return s.MaxConnections } - return defaultSqliteMaxConns + return DefaultSqliteMaxConns } // Validate checks that the SqliteConfig values are valid. From de06c29ae982adc183319025e4a3f056a56d5c06 Mon Sep 17 00:00:00 2001 From: ziggie Date: Tue, 3 Mar 2026 19:23:12 +0100 Subject: [PATCH 5/5] sqldb: add benchmark for sqlite connection number The results suggests that the default of 2 is a good conservative approach. The higher the connection number and the higher the workload the performance decreases in WAL mode since readers still need coordination (WAL index shm for example). --- sqldb/sqlite_bench_test.go | 204 +++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 sqldb/sqlite_bench_test.go diff --git a/sqldb/sqlite_bench_test.go b/sqldb/sqlite_bench_test.go new file mode 100644 index 000000000..bc5cea3ac --- /dev/null +++ b/sqldb/sqlite_bench_test.go @@ -0,0 +1,204 @@ +//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64)) + +package sqldb + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/lightningnetwork/lnd/sqldb/sqlc" + "github.com/stretchr/testify/require" +) + +// BenchmarkSqliteMaxConns benchmarks sequential reads against a SQLite +// database with varying MaxConnections settings. +// +// Run with: +// +// go test -bench=BenchmarkSqliteMaxConns -benchmem -run=^$ ./sqldb/ +func BenchmarkSqliteMaxConns(b *testing.B) { + const numInvoices = 500 + + // connCounts contains the MaxConnections values we want to compare. + // 0 means "use the library default" (currently 2 for SQLite). + connCounts := []int{1, 2, 4, 8, 16, 0} + + // Build a fresh SQLite database that will be shared across all + // sub-benchmarks. We insert a fixed set of invoices once and then + // execute read-only queries from multiple goroutines. + dbFileName := filepath.Join(b.TempDir(), "bench.db") + + // Open the store once with migrations so the schema is in place. + setupStore, err := NewSqliteStore(&SqliteConfig{ + SkipMigrations: false, + }, dbFileName) + require.NoError(b, err) + + require.NoError(b, setupStore.ApplyAllMigrations( + context.Background(), GetMigrations(), + )) + + ctx := context.Background() + + // Insert test invoices. We use a predictable hash per invoice so we + // can look them up deterministically during the benchmark. + hashes := make([][]byte, numInvoices) + for i := range numInvoices { + hash := make([]byte, 32) + hash[0] = byte(i) + hash[1] = byte(i >> 8) + hashes[i] = hash + + _, err := setupStore.InsertInvoice( + ctx, sqlc.InsertInvoiceParams{ + Hash: hash, + PaymentAddr: hash, + PaymentRequestHash: hash, + Expiry: 3600, + CreatedAt: time.Now(), + }, + ) + require.NoError(b, err) + } + + require.NoError(b, setupStore.DB.Close()) + + for _, maxConns := range connCounts { + name := fmt.Sprintf("MaxConns=%d", maxConns) + if maxConns == 0 { + name = fmt.Sprintf("MaxConns=default(%d)", + DefaultSqliteMaxConns) + } + + b.Run(name, func(b *testing.B) { + store, err := NewSqliteStore( + &SqliteConfig{ + SkipMigrations: true, + MaxConnections: maxConns, + }, dbFileName, + ) + require.NoError(b, err) + + b.Cleanup(func() { + require.NoError(b, store.DB.Close()) + }) + + var i int + for b.Loop() { + hash := hashes[i%numInvoices] + i++ + + _, err := store.GetInvoice( + ctx, + sqlc.GetInvoiceParams{ + Hash: hash, + }, + ) + if err != nil { + require.ErrorIs(b, err, sql.ErrNoRows) + } + } + }) + } +} + +// BenchmarkSqliteMaxConnsConcurrentReads measures aggregate read throughput +// for a fixed level of goroutine concurrency to complement the sequential +// benchmark above. Each iteration launches a fixed number of goroutines that +// all issue reads simultaneously, directly stressing the connection pool. +func BenchmarkSqliteMaxConnsConcurrentReads(b *testing.B) { + const ( + numInvoices = 500 + goroutines = 16 + ) + + connCounts := []int{1, 2, 4, 8, 16, 0} + + dbFileName := filepath.Join(b.TempDir(), "bench_conc.db") + + setupStore, err := NewSqliteStore(&SqliteConfig{ + SkipMigrations: false, + }, dbFileName) + require.NoError(b, err) + + require.NoError(b, setupStore.ApplyAllMigrations( + context.Background(), GetMigrations(), + )) + + ctx := context.Background() + + hashes := make([][]byte, numInvoices) + for i := range numInvoices { + hash := make([]byte, 32) + hash[0] = byte(i) + hash[1] = byte(i >> 8) + hashes[i] = hash + + _, err := setupStore.InsertInvoice( + ctx, sqlc.InsertInvoiceParams{ + Hash: hash, + PaymentAddr: hash, + PaymentRequestHash: hash, + Expiry: 3600, + CreatedAt: time.Now(), + }, + ) + require.NoError(b, err) + } + + require.NoError(b, setupStore.DB.Close()) + + for _, maxConns := range connCounts { + name := fmt.Sprintf("MaxConns=%d", maxConns) + if maxConns == 0 { + name = fmt.Sprintf("MaxConns=default(%d)", + DefaultSqliteMaxConns) + } + + b.Run(name, func(b *testing.B) { + store, err := NewSqliteStore( + &SqliteConfig{ + SkipMigrations: true, + MaxConnections: maxConns, + }, dbFileName, + ) + require.NoError(b, err) + + b.Cleanup(func() { + require.NoError(b, store.DB.Close()) + }) + + for b.Loop() { + var wg sync.WaitGroup + wg.Add(goroutines) + + for g := range goroutines { + go func() { + defer wg.Done() + + hash := hashes[g%numInvoices] + _, err := store.GetInvoice( + ctx, + sqlc.GetInvoiceParams{ + Hash: hash, + }, + ) + if err != nil && + err != sql.ErrNoRows { + + b.Errorf("GetInvoice:"+ + " %v", err) + } + }() + } + + wg.Wait() + } + }) + } +}