From 84598b6dc1573f4137c29ee7fba0ee02763bda5b Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 21 Jan 2025 17:00:15 +0100 Subject: [PATCH] sqldb: ensure schema definitions are fully SQLite compatible Previously, we applied replacements to our schema definitions to make them compatible with both SQLite and Postgres backends, as the files were not fully compatible with either. With this change, the only replacement required for SQLite has been moved to the generator script. This adjustment ensures compatibility by enabling auto-incrementing primary keys that are treated as 64-bit integers by sqlc. --- invoices/sql_migration.go | 4 +-- scripts/gen_sqlc_docker.sh | 28 +++++++++++++++++++ sqldb/postgres.go | 3 +- sqldb/sqlc/invoices.sql.go | 4 +-- sqldb/sqlc/migrations/000001_invoices.up.sql | 8 +++--- .../migrations/000003_invoice_events.up.sql | 2 +- sqldb/sqlc/models.go | 4 +-- sqldb/sqlite.go | 11 +++----- 8 files changed, 44 insertions(+), 20 deletions(-) diff --git a/invoices/sql_migration.go b/invoices/sql_migration.go index af0e4865f..0d590320d 100644 --- a/invoices/sql_migration.go +++ b/invoices/sql_migration.go @@ -100,7 +100,7 @@ func createInvoiceHashIndex(ctx context.Context, db kvdb.Backend, return tx.InsertKVInvoiceKeyAndAddIndex(ctx, sqlc.InsertKVInvoiceKeyAndAddIndexParams{ - ID: int32(invoiceKey), + ID: int64(invoiceKey), AddIndex: int64(addIndexNo), }, ) @@ -132,7 +132,7 @@ func createInvoiceHashIndex(ctx context.Context, db kvdb.Backend, return tx.SetKVInvoicePaymentHash(ctx, sqlc.SetKVInvoicePaymentHashParams{ - ID: int32(invoiceKey), + ID: int64(invoiceKey), Hash: k, }, ) diff --git a/scripts/gen_sqlc_docker.sh b/scripts/gen_sqlc_docker.sh index e011d689e..148ca05be 100755 --- a/scripts/gen_sqlc_docker.sh +++ b/scripts/gen_sqlc_docker.sh @@ -2,12 +2,40 @@ set -e +# restore_files is a function to restore original schema files. +restore_files() { + echo "Restoring SQLite bigint patch..." + for file in sqldb/sqlc/migrations/*.up.sql.bak; do + mv "$file" "${file%.bak}" + done +} + + +# Set trap to call restore_files on script exit. This makes sure the old files +# are always restored. +trap restore_files EXIT + # Directory of the script file, independent of where it's called from. DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" # Use the user's cache directories GOCACHE=`go env GOCACHE` GOMODCACHE=`go env GOMODCACHE` +# SQLite doesn't support "BIGINT PRIMARY KEY" for auto-incrementing primary +# keys, only "INTEGER PRIMARY KEY". Internally it uses 64-bit integers for +# numbers anyway, independent of the column type. So we can just use +# "INTEGER PRIMARY KEY" and it will work the same under the hood, giving us +# auto incrementing 64-bit integers. +# _BUT_, sqlc will generate Go code with int32 if we use "INTEGER PRIMARY KEY", +# even though we want int64. So before we run sqlc, we need to patch the +# source schema SQL files to use "BIGINT PRIMARY KEY" instead of "INTEGER +# PRIMARY KEY". +echo "Applying SQLite bigint patch..." +for file in sqldb/sqlc/migrations/*.up.sql; do + echo "Patching $file" + sed -i.bak -E 's/INTEGER PRIMARY KEY/BIGINT PRIMARY KEY/g' "$file" +done + echo "Generating sql models and queries in go..." docker run \ diff --git a/sqldb/postgres.go b/sqldb/postgres.go index 455ecb405..d271d214e 100644 --- a/sqldb/postgres.go +++ b/sqldb/postgres.go @@ -29,8 +29,7 @@ var ( // has some differences. postgresSchemaReplacements = map[string]string{ "BLOB": "BYTEA", - "INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY", - "BIGINT PRIMARY KEY": "BIGSERIAL PRIMARY KEY", + "INTEGER PRIMARY KEY": "BIGSERIAL PRIMARY KEY", "TIMESTAMP": "TIMESTAMP WITHOUT TIME ZONE", } diff --git a/sqldb/sqlc/invoices.sql.go b/sqldb/sqlc/invoices.sql.go index 13ac8094a..1cd7dfff4 100644 --- a/sqldb/sqlc/invoices.sql.go +++ b/sqldb/sqlc/invoices.sql.go @@ -591,7 +591,7 @@ INSERT INTO invoice_payment_hashes ( ` type InsertKVInvoiceKeyAndAddIndexParams struct { - ID int32 + ID int64 AddIndex int64 } @@ -675,7 +675,7 @@ WHERE id = $1 ` type SetKVInvoicePaymentHashParams struct { - ID int32 + ID int64 Hash []byte } diff --git a/sqldb/sqlc/migrations/000001_invoices.up.sql b/sqldb/sqlc/migrations/000001_invoices.up.sql index e4c80a26f..7f8c653d5 100644 --- a/sqldb/sqlc/migrations/000001_invoices.up.sql +++ b/sqldb/sqlc/migrations/000001_invoices.up.sql @@ -11,7 +11,7 @@ INSERT INTO invoice_sequences(name, current_value) VALUES ('settle_index', 0); -- invoices table contains all the information shared by all the invoice types. CREATE TABLE IF NOT EXISTS invoices ( -- The id of the invoice. Translates to the AddIndex. - id BIGINT PRIMARY KEY, + id INTEGER PRIMARY KEY, -- The hash for this invoice. The invoice hash will always identify that -- invoice. @@ -102,8 +102,8 @@ CREATE INDEX IF NOT EXISTS invoice_feature_invoice_id_idx ON invoice_features(in CREATE TABLE IF NOT EXISTS invoice_htlcs ( -- The id for this htlc. Used in foreign keys instead of the -- htlc_id/chan_id combination. - id BIGINT PRIMARY KEY, - + id INTEGER PRIMARY KEY, + -- Short chan id indicating the htlc's origin. uint64 stored as text. chan_id TEXT NOT NULL, @@ -111,7 +111,7 @@ CREATE TABLE IF NOT EXISTS invoice_htlcs ( -- int64 in the database. The application layer must check that there is no -- overflow when storing/loading this column. htlc_id BIGINT NOT NULL, - + -- The htlc's amount in millisatoshis. amount_msat BIGINT NOT NULL, diff --git a/sqldb/sqlc/migrations/000003_invoice_events.up.sql b/sqldb/sqlc/migrations/000003_invoice_events.up.sql index a3718c0eb..b9280a9d2 100644 --- a/sqldb/sqlc/migrations/000003_invoice_events.up.sql +++ b/sqldb/sqlc/migrations/000003_invoice_events.up.sql @@ -29,7 +29,7 @@ VALUES -- AMP sub invoices. This table can be used to create a historical view of what -- happened to the node's invoices. CREATE TABLE IF NOT EXISTS invoice_events ( - id BIGINT PRIMARY KEY, + id INTEGER PRIMARY KEY, -- added_at is the timestamp when this event was added. added_at TIMESTAMP NOT NULL, diff --git a/sqldb/sqlc/models.go b/sqldb/sqlc/models.go index f69d0352b..7d3220451 100644 --- a/sqldb/sqlc/models.go +++ b/sqldb/sqlc/models.go @@ -58,7 +58,7 @@ type InvoiceEvent struct { } type InvoiceEventType struct { - ID int32 + ID int64 Description string } @@ -88,7 +88,7 @@ type InvoiceHtlcCustomRecord struct { } type InvoicePaymentHash struct { - ID int32 + ID int64 AddIndex int64 Hash []byte } diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index 59cb03569..81e1f26b3 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -28,13 +28,10 @@ const ( ) var ( - // sqliteSchemaReplacements is a map of schema strings that need to be - // replaced for sqlite. This is needed because sqlite doesn't directly - // support the BIGINT type for primary keys, so we need to replace it - // with INTEGER. - sqliteSchemaReplacements = map[string]string{ - "BIGINT PRIMARY KEY": "INTEGER PRIMARY KEY", - } + // sqliteSchemaReplacements maps schema strings to their SQLite + // compatible replacements. Currently, no replacements are needed as our + // SQL schema definition files are designed for SQLite compatibility. + sqliteSchemaReplacements = map[string]string{} // Make sure SqliteStore implements the MigrationExecutor interface. _ MigrationExecutor = (*SqliteStore)(nil)